diff mbox series

[v3,6/7] x86: Add support for VEC_SIZE == 64 in strcmp-evex.S impl

Message ID 20221019004409.3623395-6-goldstein.w.n@gmail.com
State New
Headers show
Series [v3,1/7] x86: Optimize memchr-evex.S and implement with VMM headers | expand

Commit Message

Noah Goldstein Oct. 19, 2022, 12:44 a.m. UTC
Unused at the moment, but evex512 strcmp, strncmp, strcasecmp{l}, and
strncasecmp{l} functions can be added by including strcmp-evex.S with
"x86-evex512-vecs.h" defined.

In addition save code size a bit in a few places.

1. tzcnt ...         -> bsf ...
2. vpcmp{b|d} $0 ... -> vpcmpeq{b|d}

This saves a touch of code size but has minimal net affect.

Full check passes on x86-64.
---
 sysdeps/x86_64/multiarch/strcmp-evex.S | 676 ++++++++++++++++---------
 1 file changed, 430 insertions(+), 246 deletions(-)

Comments

H.J. Lu Oct. 19, 2022, 4:59 p.m. UTC | #1
On Tue, Oct 18, 2022 at 5:44 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> Unused at the moment, but evex512 strcmp, strncmp, strcasecmp{l}, and
> strncasecmp{l} functions can be added by including strcmp-evex.S with
> "x86-evex512-vecs.h" defined.
>
> In addition save code size a bit in a few places.
>
> 1. tzcnt ...         -> bsf ...
> 2. vpcmp{b|d} $0 ... -> vpcmpeq{b|d}
>
> This saves a touch of code size but has minimal net affect.
>
> Full check passes on x86-64.
> ---
>  sysdeps/x86_64/multiarch/strcmp-evex.S | 676 ++++++++++++++++---------
>  1 file changed, 430 insertions(+), 246 deletions(-)
>
> diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S
> index e482d0167f..756a3bb8d6 100644
> --- a/sysdeps/x86_64/multiarch/strcmp-evex.S
> +++ b/sysdeps/x86_64/multiarch/strcmp-evex.S
> @@ -20,6 +20,10 @@
>
>  #if ISA_SHOULD_BUILD (4)
>
> +# ifndef VEC_SIZE
> +#  include "x86-evex256-vecs.h"
> +# endif
> +
>  # define STRCMP_ISA    _evex
>  # include "strcmp-naming.h"
>
> @@ -35,41 +39,57 @@
>  # define PAGE_SIZE     4096
>
>         /* VEC_SIZE = Number of bytes in a ymm register.  */
> -# define VEC_SIZE      32
>  # define CHAR_PER_VEC  (VEC_SIZE       /       SIZE_OF_CHAR)
>
> -# define VMOVU vmovdqu64
> -# define VMOVA vmovdqa64
> -
>  # ifdef USE_AS_WCSCMP
> -#  define TESTEQ       subl $0xff,
>         /* Compare packed dwords.  */
>  #  define VPCMP        vpcmpd
> +#  define VPCMPEQ      vpcmpeqd
>  #  define VPMINU       vpminud
>  #  define VPTESTM      vptestmd
>  #  define VPTESTNM     vptestnmd
>         /* 1 dword char == 4 bytes.  */
>  #  define SIZE_OF_CHAR 4
> +
> +#  define TESTEQ       sub $((1 << CHAR_PER_VEC) - 1),
> +
> +#  define USE_WIDE_CHAR
>  # else
> -#  define TESTEQ       incl
>         /* Compare packed bytes.  */
>  #  define VPCMP        vpcmpb
> +#  define VPCMPEQ      vpcmpeqb
>  #  define VPMINU       vpminub
>  #  define VPTESTM      vptestmb
>  #  define VPTESTNM     vptestnmb
>         /* 1 byte char == 1 byte.  */
>  #  define SIZE_OF_CHAR 1
> +
> +#  define TESTEQ       inc
> +# endif
> +
> +# include "reg-macros.h"
> +
> +# if VEC_SIZE == 64
> +#  define RODATA_SECTION       rodata.cst64
> +# else
> +#  define RODATA_SECTION       rodata.cst32
> +# endif
> +
> +# if CHAR_PER_VEC == 64
> +#  define FALLTHROUGH_RETURN_OFFSET    (VEC_SIZE * 3)
> +# else
> +#  define FALLTHROUGH_RETURN_OFFSET    (VEC_SIZE * 2)
>  # endif
>
>  # ifdef USE_AS_STRNCMP
> -#  define LOOP_REG     r9d
> +#  define LOOP_REG     VR9
>  #  define LOOP_REG64   r9
>
>  #  define OFFSET_REG8  r9b
>  #  define OFFSET_REG   r9d
>  #  define OFFSET_REG64 r9
>  # else
> -#  define LOOP_REG     edx
> +#  define LOOP_REG     VRDX
>  #  define LOOP_REG64   rdx
>
>  #  define OFFSET_REG8  dl
> @@ -83,32 +103,6 @@
>  #  define VEC_OFFSET   (-VEC_SIZE)
>  # endif
>
> -# define XMM0  xmm17
> -# define XMM1  xmm18
> -
> -# define XMM10 xmm27
> -# define XMM11 xmm28
> -# define XMM12 xmm29
> -# define XMM13 xmm30
> -# define XMM14 xmm31
> -
> -
> -# define YMM0  ymm17
> -# define YMM1  ymm18
> -# define YMM2  ymm19
> -# define YMM3  ymm20
> -# define YMM4  ymm21
> -# define YMM5  ymm22
> -# define YMM6  ymm23
> -# define YMM7  ymm24
> -# define YMM8  ymm25
> -# define YMM9  ymm26
> -# define YMM10 ymm27
> -# define YMM11 ymm28
> -# define YMM12 ymm29
> -# define YMM13 ymm30
> -# define YMM14 ymm31
> -
>  # ifdef USE_AS_STRCASECMP_L
>  #  define BYTE_LOOP_REG        OFFSET_REG
>  # else
> @@ -125,61 +119,72 @@
>  #  endif
>  # endif
>
> -# define LCASE_MIN_YMM %YMM12
> -# define LCASE_MAX_YMM %YMM13
> -# define CASE_ADD_YMM  %YMM14
> +# define LCASE_MIN_V   VMM(12)
> +# define LCASE_MAX_V   VMM(13)
> +# define CASE_ADD_V    VMM(14)
>
> -# define LCASE_MIN_XMM %XMM12
> -# define LCASE_MAX_XMM %XMM13
> -# define CASE_ADD_XMM  %XMM14
> +# if VEC_SIZE == 64
> +#  define LCASE_MIN_YMM        VMM_256(12)
> +#  define LCASE_MAX_YMM        VMM_256(13)
> +#  define CASE_ADD_YMM VMM_256(14)
> +# endif
> +
> +# define LCASE_MIN_XMM VMM_128(12)
> +# define LCASE_MAX_XMM VMM_128(13)
> +# define CASE_ADD_XMM  VMM_128(14)
>
>         /* NB: wcsncmp uses r11 but strcasecmp is never used in
>            conjunction with wcscmp.  */
>  # define TOLOWER_BASE  %r11
>
>  # ifdef USE_AS_STRCASECMP_L
> -#  define _REG(x, y) x ## y
> -#  define REG(x, y) _REG(x, y)
> -#  define TOLOWER(reg1, reg2, ext)                                                                             \
> -       vpsubb  REG(LCASE_MIN_, ext), reg1, REG(%ext, 10);                                      \
> -       vpsubb  REG(LCASE_MIN_, ext), reg2, REG(%ext, 11);                                      \
> -       vpcmpub $1, REG(LCASE_MAX_, ext), REG(%ext, 10), %k5;                           \
> -       vpcmpub $1, REG(LCASE_MAX_, ext), REG(%ext, 11), %k6;                           \
> -       vpaddb  reg1, REG(CASE_ADD_, ext), reg1{%k5};                                           \
> -       vpaddb  reg2, REG(CASE_ADD_, ext), reg2{%k6}
> -
> -#  define TOLOWER_gpr(src, dst) movl (TOLOWER_BASE, src, 4), dst
> -#  define TOLOWER_YMM(...)     TOLOWER(__VA_ARGS__, YMM)
> -#  define TOLOWER_XMM(...)     TOLOWER(__VA_ARGS__, XMM)
> -
> -#  define CMP_R1_R2(s1_reg, s2_reg, reg_out, ext)                                              \
> -       TOLOWER (s1_reg, s2_reg, ext);                                                                          \
> -       VPCMP   $0, s1_reg, s2_reg, reg_out
> -
> -#  define CMP_R1_S2(s1_reg, s2_mem, s2_reg, reg_out, ext)                              \
> -       VMOVU   s2_mem, s2_reg;                                                                                         \
> -       CMP_R1_R2(s1_reg, s2_reg, reg_out, ext)
> -
> -#  define CMP_R1_R2_YMM(...) CMP_R1_R2(__VA_ARGS__, YMM)
> -#  define CMP_R1_R2_XMM(...) CMP_R1_R2(__VA_ARGS__, XMM)
> -
> -#  define CMP_R1_S2_YMM(...) CMP_R1_S2(__VA_ARGS__, YMM)
> -#  define CMP_R1_S2_XMM(...) CMP_R1_S2(__VA_ARGS__, XMM)
> +#  define _REG(x, y)   x ## y
> +#  define REG(x, y)    _REG(x, y)
> +#  define TOLOWER(reg1, reg2, ext, vec_macro)  \
> +       vpsubb  %REG(LCASE_MIN_, ext), reg1, %vec_macro(10);    \
> +       vpsubb  %REG(LCASE_MIN_, ext), reg2, %vec_macro(11);    \
> +       vpcmpub $1, %REG(LCASE_MAX_, ext), %vec_macro(10), %k5; \
> +       vpcmpub $1, %REG(LCASE_MAX_, ext), %vec_macro(11), %k6; \
> +       vpaddb  reg1, %REG(CASE_ADD_, ext), reg1{%k5};  \
> +       vpaddb  reg2, %REG(CASE_ADD_, ext), reg2{%k6}
> +
> +#  define TOLOWER_gpr(src, dst)        movl (TOLOWER_BASE, src, 4), dst
> +#  define TOLOWER_VMM(...)     TOLOWER(__VA_ARGS__, V, VMM)
> +#  define TOLOWER_YMM(...)     TOLOWER(__VA_ARGS__, YMM, VMM_256)
> +#  define TOLOWER_XMM(...)     TOLOWER(__VA_ARGS__, XMM, VMM_128)
> +
> +#  define CMP_R1_R2(s1_reg, s2_reg, reg_out, ext, vec_macro)   \
> +       TOLOWER (s1_reg, s2_reg, ext, vec_macro);       \
> +       VPCMPEQ s1_reg, s2_reg, reg_out
> +
> +#  define CMP_R1_S2(s1_reg, s2_mem, s2_reg, reg_out, ext, vec_macro)   \
> +       VMOVU   s2_mem, s2_reg; \
> +       CMP_R1_R2 (s1_reg, s2_reg, reg_out, ext, vec_macro)
> +
> +#  define CMP_R1_R2_VMM(...)   CMP_R1_R2(__VA_ARGS__, V, VMM)
> +#  define CMP_R1_R2_YMM(...)   CMP_R1_R2(__VA_ARGS__, YMM, VMM_256)
> +#  define CMP_R1_R2_XMM(...)   CMP_R1_R2(__VA_ARGS__, XMM, VMM_128)
> +
> +#  define CMP_R1_S2_VMM(...)   CMP_R1_S2(__VA_ARGS__, V, VMM)
> +#  define CMP_R1_S2_YMM(...)   CMP_R1_S2(__VA_ARGS__, YMM, VMM_256)
> +#  define CMP_R1_S2_XMM(...)   CMP_R1_S2(__VA_ARGS__, XMM, VMM_128)
>
>  # else
>  #  define TOLOWER_gpr(...)
> +#  define TOLOWER_VMM(...)
>  #  define TOLOWER_YMM(...)
>  #  define TOLOWER_XMM(...)
>
> -#  define CMP_R1_R2_YMM(s1_reg, s2_reg, reg_out)                                               \
> -       VPCMP   $0, s2_reg, s1_reg, reg_out
> +#  define CMP_R1_R2_VMM(s1_reg, s2_reg, reg_out)       \
> +       VPCMPEQ s2_reg, s1_reg, reg_out
>
> -#  define CMP_R1_R2_XMM(...) CMP_R1_R2_YMM(__VA_ARGS__)
> +#  define CMP_R1_R2_YMM(...)   CMP_R1_R2_VMM(__VA_ARGS__)
> +#  define CMP_R1_R2_XMM(...)   CMP_R1_R2_VMM(__VA_ARGS__)
>
> -#  define CMP_R1_S2_YMM(s1_reg, s2_mem, unused, reg_out)                               \
> -       VPCMP   $0, s2_mem, s1_reg, reg_out
> -
> -#  define CMP_R1_S2_XMM(...) CMP_R1_S2_YMM(__VA_ARGS__)
> +#  define CMP_R1_S2_VMM(s1_reg, s2_mem, unused, reg_out)       \
> +       VPCMPEQ s2_mem, s1_reg, reg_out
> +#  define CMP_R1_S2_YMM(...)   CMP_R1_S2_VMM(__VA_ARGS__)
> +#  define CMP_R1_S2_XMM(...)   CMP_R1_S2_VMM(__VA_ARGS__)
>  # endif
>
>  /* Warning!
> @@ -203,7 +208,7 @@
>     the maximum offset is reached before a difference is found, zero is
>     returned.  */
>
> -       .section .text.evex, "ax", @progbits
> +       .section SECTION(.text), "ax", @progbits
>         .align  16
>         .type   STRCMP, @function
>         .globl  STRCMP
> @@ -232,7 +237,7 @@ STRCMP:
>  #  else
>         mov     (%LOCALE_REG), %RAX_LP
>  #  endif
> -       testl   $1, LOCALE_DATA_VALUES + _NL_CTYPE_NONASCII_CASE * SIZEOF_VALUES(%rax)
> +       testb   $1, LOCALE_DATA_VALUES + _NL_CTYPE_NONASCII_CASE * SIZEOF_VALUES(%rax)
>         jne     STRCASECMP_L_NONASCII
>         leaq    _nl_C_LC_CTYPE_tolower + 128 * 4(%rip), TOLOWER_BASE
>  # endif
> @@ -254,28 +259,46 @@ STRCMP:
>  # endif
>
>  # if defined USE_AS_STRCASECMP_L
> -       .section .rodata.cst32, "aM", @progbits, 32
> -       .align  32
> +       .section RODATA_SECTION, "aM", @progbits, VEC_SIZE
> +       .align  VEC_SIZE
>  L(lcase_min):
>         .quad   0x4141414141414141
>         .quad   0x4141414141414141
>         .quad   0x4141414141414141
>         .quad   0x4141414141414141
> +#  if VEC_SIZE == 64
> +       .quad   0x4141414141414141
> +       .quad   0x4141414141414141
> +       .quad   0x4141414141414141
> +       .quad   0x4141414141414141
> +#  endif
>  L(lcase_max):
>         .quad   0x1a1a1a1a1a1a1a1a
>         .quad   0x1a1a1a1a1a1a1a1a
>         .quad   0x1a1a1a1a1a1a1a1a
>         .quad   0x1a1a1a1a1a1a1a1a
> +#  if VEC_SIZE == 64
> +       .quad   0x1a1a1a1a1a1a1a1a
> +       .quad   0x1a1a1a1a1a1a1a1a
> +       .quad   0x1a1a1a1a1a1a1a1a
> +       .quad   0x1a1a1a1a1a1a1a1a
> +#  endif
>  L(case_add):
>         .quad   0x2020202020202020
>         .quad   0x2020202020202020
>         .quad   0x2020202020202020
>         .quad   0x2020202020202020
> +#  if VEC_SIZE == 64
> +       .quad   0x2020202020202020
> +       .quad   0x2020202020202020
> +       .quad   0x2020202020202020
> +       .quad   0x2020202020202020
> +#  endif
>         .previous
>
> -       vmovdqa64 L(lcase_min)(%rip), LCASE_MIN_YMM
> -       vmovdqa64 L(lcase_max)(%rip), LCASE_MAX_YMM
> -       vmovdqa64 L(case_add)(%rip), CASE_ADD_YMM
> +       VMOVA   L(lcase_min)(%rip), %LCASE_MIN_V
> +       VMOVA   L(lcase_max)(%rip), %LCASE_MAX_V
> +       VMOVA   L(case_add)(%rip), %CASE_ADD_V
>  # endif
>
>         movl    %edi, %eax
> @@ -288,12 +311,12 @@ L(case_add):
>
>  L(no_page_cross):
>         /* Safe to compare 4x vectors.  */
> -       VMOVU   (%rdi), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> +       VMOVU   (%rdi), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
>         /* Each bit cleared in K1 represents a mismatch or a null CHAR
>            in YMM0 and 32 bytes at (%rsi).  */
> -       CMP_R1_S2_YMM (%YMM0, (%rsi), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> +       CMP_R1_S2_VMM (%VMM(0), (%rsi), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
>  # ifdef USE_AS_STRNCMP
>         cmpq    $CHAR_PER_VEC, %rdx
>         jbe     L(vec_0_test_len)
> @@ -303,14 +326,14 @@ L(no_page_cross):
>            wcscmp/wcsncmp.  */
>
>         /* All 1s represents all equals. TESTEQ will overflow to zero in
> -          all equals case. Otherwise 1s will carry until position of first
> -          mismatch.  */
> -       TESTEQ  %ecx
> +          all equals case. Otherwise 1s will carry until position of
> +          first mismatch.  */
> +       TESTEQ  %VRCX
>         jz      L(more_3x_vec)
>
>         .p2align 4,, 4
>  L(return_vec_0):
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>  # ifdef USE_AS_WCSCMP
>         movl    (%rdi, %rcx, SIZE_OF_CHAR), %edx
>         xorl    %eax, %eax
> @@ -321,7 +344,16 @@ L(return_vec_0):
>         orl     $1, %eax
>  # else
>         movzbl  (%rdi, %rcx), %eax
> +       /* For VEC_SIZE == 64 use movb instead of movzbl to save a byte
> +          and keep logic for len <= VEC_SIZE (common) in just the
> +          first cache line.  NB: No evex512 processor has partial-
> +          register stalls. If that changes this ifdef can be disabled
> +          without affecting correctness.  */
> +#  if !defined USE_AS_STRNCMP && !defined USE_AS_STRCASECMP_L && VEC_SIZE == 64
> +       movb    (%rsi, %rcx), %cl
> +#  else
>         movzbl  (%rsi, %rcx), %ecx
> +#  endif
>         TOLOWER_gpr (%rax, %eax)
>         TOLOWER_gpr (%rcx, %ecx)
>         subl    %ecx, %eax
> @@ -332,8 +364,8 @@ L(ret0):
>  # ifdef USE_AS_STRNCMP
>         .p2align 4,, 4
>  L(vec_0_test_len):
> -       notl    %ecx
> -       bzhil   %edx, %ecx, %eax
> +       not     %VRCX
> +       bzhi    %VRDX, %VRCX, %VRAX
>         jnz     L(return_vec_0)
>         /* Align if will cross fetch block.  */
>         .p2align 4,, 2
> @@ -372,7 +404,7 @@ L(ret1):
>
>         .p2align 4,, 10
>  L(return_vec_1):
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>  # ifdef USE_AS_STRNCMP
>         /* rdx must be > CHAR_PER_VEC so its safe to subtract without
>            worrying about underflow.  */
> @@ -401,24 +433,41 @@ L(ret2):
>         .p2align 4,, 10
>  # ifdef USE_AS_STRNCMP
>  L(return_vec_3):
> -#  if CHAR_PER_VEC <= 16
> +#  if CHAR_PER_VEC <= 32
> +       /* If CHAR_PER_VEC <= 32 reuse code from L(return_vec_3) without
> +          additional branches by adjusting the bit positions from
> +          VEC3.  We can't do this for CHAR_PER_VEC == 64.  */
> +#   if CHAR_PER_VEC <= 16
>         sall    $CHAR_PER_VEC, %ecx
> -#  else
> +#   else
>         salq    $CHAR_PER_VEC, %rcx
> +#   endif
> +#  else
> +       /* If CHAR_PER_VEC == 64 we can't shift the return GPR so just
> +          check it.  */
> +       bsf     %VRCX, %VRCX
> +       addl    $(CHAR_PER_VEC), %ecx
> +       cmpq    %rcx, %rdx
> +       ja      L(ret_vec_3_finish)
> +       xorl    %eax, %eax
> +       ret
>  #  endif
>  # endif
> +
> +       /* If CHAR_PER_VEC == 64 we can't combine matches from the last
> +          2x VEC so need seperate return label.  */
>  L(return_vec_2):
>  # if (CHAR_PER_VEC <= 16) || !(defined USE_AS_STRNCMP)
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>  # else
> -       tzcntq  %rcx, %rcx
> +       bsfq    %rcx, %rcx
>  # endif
> -
>  # ifdef USE_AS_STRNCMP
>         cmpq    %rcx, %rdx
>         jbe     L(ret_zero)
>  # endif
>
> +L(ret_vec_3_finish):
>  # ifdef USE_AS_WCSCMP
>         movl    (VEC_SIZE * 2)(%rdi, %rcx, SIZE_OF_CHAR), %edx
>         xorl    %eax, %eax
> @@ -440,7 +489,7 @@ L(ret3):
>  # ifndef USE_AS_STRNCMP
>         .p2align 4,, 10
>  L(return_vec_3):
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>  #  ifdef USE_AS_WCSCMP
>         movl    (VEC_SIZE * 3)(%rdi, %rcx, SIZE_OF_CHAR), %edx
>         xorl    %eax, %eax
> @@ -465,11 +514,11 @@ L(ret4):
>         .p2align 5
>  L(more_3x_vec):
>         /* Safe to compare 4x vectors.  */
> -       VMOVU   (VEC_SIZE)(%rdi), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, VEC_SIZE(%rsi), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVU   (VEC_SIZE)(%rdi), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), VEC_SIZE(%rsi), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_1)
>
>  # ifdef USE_AS_STRNCMP
> @@ -477,18 +526,18 @@ L(more_3x_vec):
>         jbe     L(ret_zero)
>  # endif
>
> -       VMOVU   (VEC_SIZE * 2)(%rdi), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, (VEC_SIZE * 2)(%rsi), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVU   (VEC_SIZE * 2)(%rdi), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), (VEC_SIZE * 2)(%rsi), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_2)
>
> -       VMOVU   (VEC_SIZE * 3)(%rdi), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, (VEC_SIZE * 3)(%rsi), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVU   (VEC_SIZE * 3)(%rdi), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), (VEC_SIZE * 3)(%rsi), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_3)
>
>  # ifdef USE_AS_STRNCMP
> @@ -565,110 +614,123 @@ L(loop):
>
>         /* Loop entry after handling page cross during loop.  */
>  L(loop_skip_page_cross_check):
> -       VMOVA   (VEC_SIZE * 0)(%rdi), %YMM0
> -       VMOVA   (VEC_SIZE * 1)(%rdi), %YMM2
> -       VMOVA   (VEC_SIZE * 2)(%rdi), %YMM4
> -       VMOVA   (VEC_SIZE * 3)(%rdi), %YMM6
> +       VMOVA   (VEC_SIZE * 0)(%rdi), %VMM(0)
> +       VMOVA   (VEC_SIZE * 1)(%rdi), %VMM(2)
> +       VMOVA   (VEC_SIZE * 2)(%rdi), %VMM(4)
> +       VMOVA   (VEC_SIZE * 3)(%rdi), %VMM(6)
>
> -       VPMINU  %YMM0, %YMM2, %YMM8
> -       VPMINU  %YMM4, %YMM6, %YMM9
> +       VPMINU  %VMM(0), %VMM(2), %VMM(8)
> +       VPMINU  %VMM(4), %VMM(6), %VMM(9)
>
>         /* A zero CHAR in YMM9 means that there is a null CHAR.  */
> -       VPMINU  %YMM8, %YMM9, %YMM9
> +       VPMINU  %VMM(8), %VMM(9), %VMM(9)
>
>         /* Each bit set in K1 represents a non-null CHAR in YMM9.  */
> -       VPTESTM %YMM9, %YMM9, %k1
> +       VPTESTM %VMM(9), %VMM(9), %k1
>  # ifndef USE_AS_STRCASECMP_L
> -       vpxorq  (VEC_SIZE * 0)(%rsi), %YMM0, %YMM1
> -       vpxorq  (VEC_SIZE * 1)(%rsi), %YMM2, %YMM3
> -       vpxorq  (VEC_SIZE * 2)(%rsi), %YMM4, %YMM5
> +       vpxorq  (VEC_SIZE * 0)(%rsi), %VMM(0), %VMM(1)
> +       vpxorq  (VEC_SIZE * 1)(%rsi), %VMM(2), %VMM(3)
> +       vpxorq  (VEC_SIZE * 2)(%rsi), %VMM(4), %VMM(5)
>         /* Ternary logic to xor (VEC_SIZE * 3)(%rsi) with YMM6 while
>            oring with YMM1. Result is stored in YMM6.  */
> -       vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %YMM1, %YMM6
> +       vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %VMM(1), %VMM(6)
>  # else
> -       VMOVU   (VEC_SIZE * 0)(%rsi), %YMM1
> -       TOLOWER_YMM (%YMM0, %YMM1)
> -       VMOVU   (VEC_SIZE * 1)(%rsi), %YMM3
> -       TOLOWER_YMM (%YMM2, %YMM3)
> -       VMOVU   (VEC_SIZE * 2)(%rsi), %YMM5
> -       TOLOWER_YMM (%YMM4, %YMM5)
> -       VMOVU   (VEC_SIZE * 3)(%rsi), %YMM7
> -       TOLOWER_YMM (%YMM6, %YMM7)
> -       vpxorq  %YMM0, %YMM1, %YMM1
> -       vpxorq  %YMM2, %YMM3, %YMM3
> -       vpxorq  %YMM4, %YMM5, %YMM5
> -       vpternlogd $0xde, %YMM7, %YMM1, %YMM6
> +       VMOVU   (VEC_SIZE * 0)(%rsi), %VMM(1)
> +       TOLOWER_VMM (%VMM(0), %VMM(1))
> +       VMOVU   (VEC_SIZE * 1)(%rsi), %VMM(3)
> +       TOLOWER_VMM (%VMM(2), %VMM(3))
> +       VMOVU   (VEC_SIZE * 2)(%rsi), %VMM(5)
> +       TOLOWER_VMM (%VMM(4), %VMM(5))
> +       VMOVU   (VEC_SIZE * 3)(%rsi), %VMM(7)
> +       TOLOWER_VMM (%VMM(6), %VMM(7))
> +       vpxorq  %VMM(0), %VMM(1), %VMM(1)
> +       vpxorq  %VMM(2), %VMM(3), %VMM(3)
> +       vpxorq  %VMM(4), %VMM(5), %VMM(5)
> +       vpternlogd $0xde, %VMM(7), %VMM(1), %VMM(6)
>  # endif
>         /* Or together YMM3, YMM5, and YMM6.  */
> -       vpternlogd $0xfe, %YMM3, %YMM5, %YMM6
> +       vpternlogd $0xfe, %VMM(3), %VMM(5), %VMM(6)
>
>
>         /* A non-zero CHAR in YMM6 represents a mismatch.  */
> -       VPTESTNM %YMM6, %YMM6, %k0{%k1}
> -       kmovd   %k0, %LOOP_REG
> +       VPTESTNM %VMM(6), %VMM(6), %k0{%k1}
> +       KMOV    %k0, %LOOP_REG
>
>         TESTEQ  %LOOP_REG
>         jz      L(loop)
>
>
>         /* Find which VEC has the mismatch of end of string.  */
> -       VPTESTM %YMM0, %YMM0, %k1
> -       VPTESTNM %YMM1, %YMM1, %k0{%k1}
> -       kmovd   %k0, %ecx
> -       TESTEQ  %ecx
> +       VPTESTM %VMM(0), %VMM(0), %k1
> +       VPTESTNM %VMM(1), %VMM(1), %k0{%k1}
> +       KMOV    %k0, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_0_end)
>
> -       VPTESTM %YMM2, %YMM2, %k1
> -       VPTESTNM %YMM3, %YMM3, %k0{%k1}
> -       kmovd   %k0, %ecx
> -       TESTEQ  %ecx
> +       VPTESTM %VMM(2), %VMM(2), %k1
> +       VPTESTNM %VMM(3), %VMM(3), %k0{%k1}
> +       KMOV    %k0, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_1_end)
>
>
> -       /* Handle VEC 2 and 3 without branches.  */
> +       /* Handle VEC 2 and 3 without branches if CHAR_PER_VEC <= 32.
> +        */
>  L(return_vec_2_3_end):
>  # ifdef USE_AS_STRNCMP
>         subq    $(CHAR_PER_VEC * 2), %rdx
>         jbe     L(ret_zero_end)
>  # endif
>
> -       VPTESTM %YMM4, %YMM4, %k1
> -       VPTESTNM %YMM5, %YMM5, %k0{%k1}
> -       kmovd   %k0, %ecx
> -       TESTEQ  %ecx
> +       VPTESTM %VMM(4), %VMM(4), %k1
> +       VPTESTNM %VMM(5), %VMM(5), %k0{%k1}
> +       KMOV    %k0, %VRCX
> +       TESTEQ  %VRCX
>  # if CHAR_PER_VEC <= 16
>         sall    $CHAR_PER_VEC, %LOOP_REG
>         orl     %ecx, %LOOP_REG
> -# else
> +# elif CHAR_PER_VEC <= 32
>         salq    $CHAR_PER_VEC, %LOOP_REG64
>         orq     %rcx, %LOOP_REG64
> +# else
> +       /* We aren't combining last 2x VEC so branch on second the last.
> +        */
> +       jnz     L(return_vec_2_end)
>  # endif
> -L(return_vec_3_end):
> +
>         /* LOOP_REG contains matches for null/mismatch from the loop. If
> -          VEC 0,1,and 2 all have no null and no mismatches then mismatch
> -          must entirely be from VEC 3 which is fully represented by
> -          LOOP_REG.  */
> +          VEC 0,1,and 2 all have no null and no mismatches then
> +          mismatch must entirely be from VEC 3 which is fully
> +          represented by LOOP_REG.  */
>  # if CHAR_PER_VEC <= 16
> -       tzcntl  %LOOP_REG, %LOOP_REG
> +       bsf     %LOOP_REG, %LOOP_REG
>  # else
> -       tzcntq  %LOOP_REG64, %LOOP_REG64
> +       bsfq    %LOOP_REG64, %LOOP_REG64
>  # endif
>  # ifdef USE_AS_STRNCMP
> +
> +       /* If CHAR_PER_VEC == 64 we can't combine last 2x VEC so need to
> +          adj length before last comparison.  */
> +#  if CHAR_PER_VEC == 64
> +       subq    $CHAR_PER_VEC, %rdx
> +       jbe     L(ret_zero_end)
> +#  endif
> +
>         cmpq    %LOOP_REG64, %rdx
>         jbe     L(ret_zero_end)
>  # endif
>
>  # ifdef USE_AS_WCSCMP
> -       movl    (VEC_SIZE * 2)(%rdi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
> +       movl    (FALLTHROUGH_RETURN_OFFSET)(%rdi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
>         xorl    %eax, %eax
> -       cmpl    (VEC_SIZE * 2)(%rsi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
> +       cmpl    (FALLTHROUGH_RETURN_OFFSET)(%rsi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
>         je      L(ret5)
>         setl    %al
>         negl    %eax
>         xorl    %r8d, %eax
>  # else
> -       movzbl  (VEC_SIZE * 2)(%rdi, %LOOP_REG64), %eax
> -       movzbl  (VEC_SIZE * 2)(%rsi, %LOOP_REG64), %ecx
> +       movzbl  (FALLTHROUGH_RETURN_OFFSET)(%rdi, %LOOP_REG64), %eax
> +       movzbl  (FALLTHROUGH_RETURN_OFFSET)(%rsi, %LOOP_REG64), %ecx
>         TOLOWER_gpr (%rax, %eax)
>         TOLOWER_gpr (%rcx, %ecx)
>         subl    %ecx, %eax
> @@ -686,23 +748,39 @@ L(ret_zero_end):
>  # endif
>
>
> +
>         /* The L(return_vec_N_end) differ from L(return_vec_N) in that
> -          they use the value of `r8` to negate the return value. This is
> -          because the page cross logic can swap `rdi` and `rsi`.  */
> +          they use the value of `r8` to negate the return value. This
> +          is because the page cross logic can swap `rdi` and `rsi`.
> +        */
>         .p2align 4,, 10
>  # ifdef USE_AS_STRNCMP
>  L(return_vec_1_end):
> -#  if CHAR_PER_VEC <= 16
> +#  if CHAR_PER_VEC <= 32
> +       /* If CHAR_PER_VEC <= 32 reuse code from L(return_vec_0_end)
> +          without additional branches by adjusting the bit positions
> +          from VEC1.  We can't do this for CHAR_PER_VEC == 64.  */
> +#   if CHAR_PER_VEC <= 16
>         sall    $CHAR_PER_VEC, %ecx
> -#  else
> +#   else
>         salq    $CHAR_PER_VEC, %rcx
> +#   endif
> +#  else
> +       /* If CHAR_PER_VEC == 64 we can't shift the return GPR so just
> +          check it.  */
> +       bsf     %VRCX, %VRCX
> +       addl    $(CHAR_PER_VEC), %ecx
> +       cmpq    %rcx, %rdx
> +       ja      L(ret_vec_0_end_finish)
> +       xorl    %eax, %eax
> +       ret
>  #  endif
>  # endif
>  L(return_vec_0_end):
>  # if (CHAR_PER_VEC <= 16) || !(defined USE_AS_STRNCMP)
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>  # else
> -       tzcntq  %rcx, %rcx
> +       bsfq    %rcx, %rcx
>  # endif
>
>  # ifdef USE_AS_STRNCMP
> @@ -710,6 +788,7 @@ L(return_vec_0_end):
>         jbe     L(ret_zero_end)
>  # endif
>
> +L(ret_vec_0_end_finish):
>  # ifdef USE_AS_WCSCMP
>         movl    (%rdi, %rcx, SIZE_OF_CHAR), %edx
>         xorl    %eax, %eax
> @@ -737,7 +816,7 @@ L(ret6):
>  # ifndef USE_AS_STRNCMP
>         .p2align 4,, 10
>  L(return_vec_1_end):
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>  #  ifdef USE_AS_WCSCMP
>         movl    VEC_SIZE(%rdi, %rcx, SIZE_OF_CHAR), %edx
>         xorl    %eax, %eax
> @@ -760,6 +839,41 @@ L(ret7):
>  # endif
>
>
> +       /* If CHAR_PER_VEC == 64 we can't combine matches from the last
> +          2x VEC so need seperate return label.  */
> +# if CHAR_PER_VEC == 64
> +L(return_vec_2_end):
> +       bsf     %VRCX, %VRCX
> +#  ifdef USE_AS_STRNCMP
> +       cmpq    %rcx, %rdx
> +       jbe     L(ret_zero_end)
> +#  endif
> +#  ifdef USE_AS_WCSCMP
> +       movl    (VEC_SIZE * 2)(%rdi, %rcx, SIZE_OF_CHAR), %edx
> +       xorl    %eax, %eax
> +       cmpl    (VEC_SIZE * 2)(%rsi, %rcx, SIZE_OF_CHAR), %edx
> +       je      L(ret31)
> +       setl    %al
> +       negl    %eax
> +       /* This is the non-zero case for `eax` so just xorl with `r8d`
> +          flip is `rdi` and `rsi` where swapped.  */
> +       xorl    %r8d, %eax
> +#  else
> +       movzbl  (VEC_SIZE * 2)(%rdi, %rcx), %eax
> +       movzbl  (VEC_SIZE * 2)(%rsi, %rcx), %ecx
> +       TOLOWER_gpr (%rax, %eax)
> +       TOLOWER_gpr (%rcx, %ecx)
> +       subl    %ecx, %eax
> +       /* Flip `eax` if `rdi` and `rsi` where swapped in page cross
> +          logic. Subtract `r8d` after xor for zero case.  */
> +       xorl    %r8d, %eax
> +       subl    %r8d, %eax
> +#  endif
> +L(ret13):
> +       ret
> +# endif
> +
> +
>         /* Page cross in rsi in next 4x VEC.  */
>
>         /* TODO: Improve logic here.  */
> @@ -778,11 +892,11 @@ L(page_cross_during_loop):
>         cmpl    $-(VEC_SIZE * 3), %eax
>         jle     L(less_1x_vec_till_page_cross)
>
> -       VMOVA   (%rdi), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, (%rsi), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVA   (%rdi), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), (%rsi), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_0_end)
>
>         /* if distance >= 2x VEC then eax > -(VEC_SIZE * 2).  */
> @@ -799,9 +913,9 @@ L(less_1x_vec_till_page_cross):
>            to read back -VEC_SIZE. If rdi is truly at the start of a page
>            here, it means the previous page (rdi - VEC_SIZE) has already
>            been loaded earlier so must be valid.  */
> -       VMOVU   -VEC_SIZE(%rdi, %rax), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, -VEC_SIZE(%rsi, %rax), %YMM1, %k1){%k2}
> +       VMOVU   -VEC_SIZE(%rdi, %rax), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), -VEC_SIZE(%rsi, %rax), %VMM(1), %k1){%k2}
>         /* Mask of potentially valid bits. The lower bits can be out of
>            range comparisons (but safe regarding page crosses).  */
>
> @@ -813,12 +927,12 @@ L(less_1x_vec_till_page_cross):
>         shlxl   %ecx, %r10d, %ecx
>         movzbl  %cl, %r10d
>  # else
> -       movl    $-1, %ecx
> -       shlxl   %esi, %ecx, %r10d
> +       mov     $-1, %VRCX
> +       shlx    %VRSI, %VRCX, %VR10
>  # endif
>
> -       kmovd   %k1, %ecx
> -       notl    %ecx
> +       KMOV    %k1, %VRCX
> +       not     %VRCX
>
>
>  # ifdef USE_AS_STRNCMP
> @@ -838,12 +952,10 @@ L(less_1x_vec_till_page_cross):
>         /* Readjust eax before potentially returning to the loop.  */
>         addl    $(PAGE_SIZE - VEC_SIZE * 4), %eax
>
> -       andl    %r10d, %ecx
> +       and     %VR10, %VRCX
>         jz      L(loop_skip_page_cross_check)
>
> -       .p2align 4,, 3
> -L(return_page_cross_end):
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>
>  # if (defined USE_AS_STRNCMP) || (defined USE_AS_WCSCMP)
>         leal    -VEC_SIZE(%OFFSET_REG64, %rcx, SIZE_OF_CHAR), %ecx
> @@ -874,8 +986,12 @@ L(ret8):
>  # ifdef USE_AS_STRNCMP
>         .p2align 4,, 10
>  L(return_page_cross_end_check):
> -       andl    %r10d, %ecx
> -       tzcntl  %ecx, %ecx
> +       and     %VR10, %VRCX
> +       /* Need to use tzcnt here as VRCX may be zero.  If VRCX is zero
> +          tzcnt(VRCX) will be CHAR_PER and remaining length (edx) is
> +          guranteed to be <= CHAR_PER_VEC so we will only use the return
> +          idx if VRCX was non-zero.  */
> +       tzcnt   %VRCX, %VRCX
>         leal    -VEC_SIZE(%rax, %rcx, SIZE_OF_CHAR), %ecx
>  #  ifdef USE_AS_WCSCMP
>         sall    $2, %edx
> @@ -892,11 +1008,11 @@ L(more_2x_vec_till_page_cross):
>         /* If more 2x vec till cross we will complete a full loop
>            iteration here.  */
>
> -       VMOVA   VEC_SIZE(%rdi), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, VEC_SIZE(%rsi), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVA   VEC_SIZE(%rdi), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), VEC_SIZE(%rsi), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_1_end)
>
>  # ifdef USE_AS_STRNCMP
> @@ -907,18 +1023,18 @@ L(more_2x_vec_till_page_cross):
>         subl    $-(VEC_SIZE * 4), %eax
>
>         /* Safe to include comparisons from lower bytes.  */
> -       VMOVU   -(VEC_SIZE * 2)(%rdi, %rax), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, -(VEC_SIZE * 2)(%rsi, %rax), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVU   -(VEC_SIZE * 2)(%rdi, %rax), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), -(VEC_SIZE * 2)(%rsi, %rax), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_page_cross_0)
>
> -       VMOVU   -(VEC_SIZE * 1)(%rdi, %rax), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, -(VEC_SIZE * 1)(%rsi, %rax), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVU   -(VEC_SIZE * 1)(%rdi, %rax), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), -(VEC_SIZE * 1)(%rsi, %rax), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(return_vec_page_cross_1)
>
>  # ifdef USE_AS_STRNCMP
> @@ -937,30 +1053,30 @@ L(more_2x_vec_till_page_cross):
>  # endif
>
>         /* Finish the loop.  */
> -       VMOVA   (VEC_SIZE * 2)(%rdi), %YMM4
> -       VMOVA   (VEC_SIZE * 3)(%rdi), %YMM6
> -       VPMINU  %YMM4, %YMM6, %YMM9
> -       VPTESTM %YMM9, %YMM9, %k1
> +       VMOVA   (VEC_SIZE * 2)(%rdi), %VMM(4)
> +       VMOVA   (VEC_SIZE * 3)(%rdi), %VMM(6)
> +       VPMINU  %VMM(4), %VMM(6), %VMM(9)
> +       VPTESTM %VMM(9), %VMM(9), %k1
>  # ifndef USE_AS_STRCASECMP_L
> -       vpxorq  (VEC_SIZE * 2)(%rsi), %YMM4, %YMM5
> +       vpxorq  (VEC_SIZE * 2)(%rsi), %VMM(4), %VMM(5)
>         /* YMM6 = YMM5 | ((VEC_SIZE * 3)(%rsi) ^ YMM6).  */
> -       vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %YMM5, %YMM6
> +       vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %VMM(5), %VMM(6)
>  # else
> -       VMOVU   (VEC_SIZE * 2)(%rsi), %YMM5
> -       TOLOWER_YMM (%YMM4, %YMM5)
> -       VMOVU   (VEC_SIZE * 3)(%rsi), %YMM7
> -       TOLOWER_YMM (%YMM6, %YMM7)
> -       vpxorq  %YMM4, %YMM5, %YMM5
> -       vpternlogd $0xde, %YMM7, %YMM5, %YMM6
> -# endif
> -       VPTESTNM %YMM6, %YMM6, %k0{%k1}
> -       kmovd   %k0, %LOOP_REG
> +       VMOVU   (VEC_SIZE * 2)(%rsi), %VMM(5)
> +       TOLOWER_VMM (%VMM(4), %VMM(5))
> +       VMOVU   (VEC_SIZE * 3)(%rsi), %VMM(7)
> +       TOLOWER_VMM (%VMM(6), %VMM(7))
> +       vpxorq  %VMM(4), %VMM(5), %VMM(5)
> +       vpternlogd $0xde, %VMM(7), %VMM(5), %VMM(6)
> +# endif
> +       VPTESTNM %VMM(6), %VMM(6), %k0{%k1}
> +       KMOV    %k0, %LOOP_REG
>         TESTEQ  %LOOP_REG
>         jnz     L(return_vec_2_3_end)
>
>         /* Best for code size to include ucond-jmp here. Would be faster
> -          if this case is hot to duplicate the L(return_vec_2_3_end) code
> -          as fall-through and have jump back to loop on mismatch
> +          if this case is hot to duplicate the L(return_vec_2_3_end)
> +          code as fall-through and have jump back to loop on mismatch
>            comparison.  */
>         subq    $-(VEC_SIZE * 4), %rdi
>         subq    $-(VEC_SIZE * 4), %rsi
> @@ -980,7 +1096,7 @@ L(ret_zero_in_loop_page_cross):
>  L(return_vec_page_cross_0):
>         addl    $-VEC_SIZE, %eax
>  L(return_vec_page_cross_1):
> -       tzcntl  %ecx, %ecx
> +       bsf     %VRCX, %VRCX
>  # if defined USE_AS_STRNCMP || defined USE_AS_WCSCMP
>         leal    -VEC_SIZE(%rax, %rcx, SIZE_OF_CHAR), %ecx
>  #  ifdef USE_AS_STRNCMP
> @@ -1023,8 +1139,8 @@ L(ret9):
>  L(page_cross):
>  # ifndef USE_AS_STRNCMP
>         /* If both are VEC aligned we don't need any special logic here.
> -          Only valid for strcmp where stop condition is guranteed to be
> -          reachable by just reading memory.  */
> +          Only valid for strcmp where stop condition is guranteed to
> +          be reachable by just reading memory.  */
>         testl   $((VEC_SIZE - 1) << 20), %eax
>         jz      L(no_page_cross)
>  # endif
> @@ -1065,11 +1181,11 @@ L(page_cross):
>            loadable memory until within 1x VEC of page cross.  */
>         .p2align 4,, 8
>  L(page_cross_loop):
> -       VMOVU   (%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM1, %k1){%k2}
> -       kmovd   %k1, %ecx
> -       TESTEQ  %ecx
> +       VMOVU   (%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(1), %k1){%k2}
> +       KMOV    %k1, %VRCX
> +       TESTEQ  %VRCX
>         jnz     L(check_ret_vec_page_cross)
>         addl    $CHAR_PER_VEC, %OFFSET_REG
>  # ifdef USE_AS_STRNCMP
> @@ -1087,13 +1203,13 @@ L(page_cross_loop):
>         subl    %eax, %OFFSET_REG
>         /* OFFSET_REG has distance to page cross - VEC_SIZE. Guranteed
>            to not cross page so is safe to load. Since we have already
> -          loaded at least 1 VEC from rsi it is also guranteed to be safe.
> -        */
> -       VMOVU   (%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM0
> -       VPTESTM %YMM0, %YMM0, %k2
> -       CMP_R1_S2_YMM (%YMM0, (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM1, %k1){%k2}
> +          loaded at least 1 VEC from rsi it is also guranteed to be
> +          safe.  */
> +       VMOVU   (%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(0)
> +       VPTESTM %VMM(0), %VMM(0), %k2
> +       CMP_R1_S2_VMM (%VMM(0), (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(1), %k1){%k2}
>
> -       kmovd   %k1, %ecx
> +       KMOV    %k1, %VRCX
>  # ifdef USE_AS_STRNCMP
>         leal    CHAR_PER_VEC(%OFFSET_REG64), %eax
>         cmpq    %rax, %rdx
> @@ -1104,7 +1220,7 @@ L(page_cross_loop):
>         addq    %rdi, %rdx
>  #  endif
>  # endif
> -       TESTEQ  %ecx
> +       TESTEQ  %VRCX
>         jz      L(prepare_loop_no_len)
>
>         .p2align 4,, 4
> @@ -1112,7 +1228,7 @@ L(ret_vec_page_cross):
>  # ifndef USE_AS_STRNCMP
>  L(check_ret_vec_page_cross):
>  # endif
> -       tzcntl  %ecx, %ecx
> +       tzcnt   %VRCX, %VRCX
>         addl    %OFFSET_REG, %ecx
>  L(ret_vec_page_cross_cont):
>  # ifdef USE_AS_WCSCMP
> @@ -1139,9 +1255,9 @@ L(ret12):
>  # ifdef USE_AS_STRNCMP
>         .p2align 4,, 10
>  L(check_ret_vec_page_cross2):
> -       TESTEQ  %ecx
> +       TESTEQ  %VRCX
>  L(check_ret_vec_page_cross):
> -       tzcntl  %ecx, %ecx
> +       tzcnt   %VRCX, %VRCX
>         addl    %OFFSET_REG, %ecx
>         cmpq    %rcx, %rdx
>         ja      L(ret_vec_page_cross_cont)
> @@ -1180,8 +1296,71 @@ L(less_1x_vec_till_page):
>  # ifdef USE_AS_WCSCMP
>         shrl    $2, %eax
>  # endif
> +
> +       /* Find largest load size we can use. VEC_SIZE == 64 only check
> +          if we can do a full ymm load.  */
> +# if VEC_SIZE == 64
> +
> +       cmpl    $((VEC_SIZE - 32) / SIZE_OF_CHAR), %eax
> +       ja      L(less_32_till_page)
> +
> +
> +       /* Use 16 byte comparison.  */
> +       VMOVU   (%rdi), %VMM_256(0)
> +       VPTESTM %VMM_256(0), %VMM_256(0), %k2
> +       CMP_R1_S2_YMM (%VMM_256(0), (%rsi), %VMM_256(1), %k1){%k2}
> +       kmovd   %k1, %ecx
> +#  ifdef USE_AS_WCSCMP
> +       subl    $0xff, %ecx
> +#  else
> +       incl    %ecx
> +#  endif
> +       jnz     L(check_ret_vec_page_cross)
> +       movl    $((VEC_SIZE - 32) / SIZE_OF_CHAR), %OFFSET_REG
> +#  ifdef USE_AS_STRNCMP
> +       cmpq    %OFFSET_REG64, %rdx
> +       jbe     L(ret_zero_page_cross_slow_case64)
> +       subl    %eax, %OFFSET_REG
> +#  else
> +       /* Explicit check for 32 byte alignment.  */
> +       subl    %eax, %OFFSET_REG
> +       jz      L(prepare_loop)
> +#  endif
> +       VMOVU   (%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM_256(0)
> +       VPTESTM %VMM_256(0), %VMM_256(0), %k2
> +       CMP_R1_S2_YMM (%VMM_256(0), (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM_256(1), %k1){%k2}
> +       kmovd   %k1, %ecx
> +#  ifdef USE_AS_WCSCMP
> +       subl    $0xff, %ecx
> +#  else
> +       incl    %ecx
> +#  endif
> +       jnz     L(check_ret_vec_page_cross)
> +#  ifdef USE_AS_STRNCMP
> +       addl    $(32 / SIZE_OF_CHAR), %OFFSET_REG
> +       subq    %OFFSET_REG64, %rdx
> +       jbe     L(ret_zero_page_cross_slow_case64)
> +       subq    $-(CHAR_PER_VEC * 4), %rdx
> +
> +       leaq    -(VEC_SIZE * 4)(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %rdi
> +       leaq    -(VEC_SIZE * 4)(%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %rsi
> +#  else
> +       leaq    (32 - VEC_SIZE * 4)(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %rdi
> +       leaq    (32 - VEC_SIZE * 4)(%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %rsi
> +#  endif
> +       jmp     L(prepare_loop_aligned)
> +
> +#  ifdef USE_AS_STRNCMP
> +       .p2align 4,, 2
> +L(ret_zero_page_cross_slow_case64):
> +       xorl    %eax, %eax
> +       ret
> +#  endif
> +L(less_32_till_page):
> +# endif
> +
>         /* Find largest load size we can use.  */
> -       cmpl    $(16 / SIZE_OF_CHAR), %eax
> +       cmpl    $((VEC_SIZE - 16) / SIZE_OF_CHAR), %eax
>         ja      L(less_16_till_page)
>
>         /* Use 16 byte comparison.  */
> @@ -1195,9 +1374,14 @@ L(less_1x_vec_till_page):
>         incw    %cx
>  # endif
>         jnz     L(check_ret_vec_page_cross)
> -       movl    $(16 / SIZE_OF_CHAR), %OFFSET_REG
> +
> +       movl    $((VEC_SIZE - 16) / SIZE_OF_CHAR), %OFFSET_REG
>  # ifdef USE_AS_STRNCMP
> +#  if VEC_SIZE == 32
>         cmpq    %OFFSET_REG64, %rdx
> +#  else
> +       cmpq    $(16 / SIZE_OF_CHAR), %rdx
> +#  endif
>         jbe     L(ret_zero_page_cross_slow_case0)
>         subl    %eax, %OFFSET_REG
>  # else
> @@ -1239,7 +1423,7 @@ L(ret_zero_page_cross_slow_case0):
>
>         .p2align 4,, 10
>  L(less_16_till_page):
> -       cmpl    $(24 / SIZE_OF_CHAR), %eax
> +       cmpl    $((VEC_SIZE - 8) / SIZE_OF_CHAR), %eax
>         ja      L(less_8_till_page)
>
>         /* Use 8 byte comparison.  */
> @@ -1260,7 +1444,7 @@ L(less_16_till_page):
>         cmpq    $(8 / SIZE_OF_CHAR), %rdx
>         jbe     L(ret_zero_page_cross_slow_case0)
>  # endif
> -       movl    $(24 / SIZE_OF_CHAR), %OFFSET_REG
> +       movl    $((VEC_SIZE - 8) / SIZE_OF_CHAR), %OFFSET_REG
>         subl    %eax, %OFFSET_REG
>
>         vmovq   (%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %xmm0
> @@ -1320,7 +1504,7 @@ L(ret_less_8_wcs):
>         ret
>
>  # else
> -       cmpl    $28, %eax
> +       cmpl    $(VEC_SIZE - 4), %eax
>         ja      L(less_4_till_page)
>
>         vmovd   (%rdi), %xmm0
> @@ -1335,7 +1519,7 @@ L(ret_less_8_wcs):
>         cmpq    $4, %rdx
>         jbe     L(ret_zero_page_cross_slow_case1)
>  #  endif
> -       movl    $(28 / SIZE_OF_CHAR), %OFFSET_REG
> +       movl    $((VEC_SIZE - 4) / SIZE_OF_CHAR), %OFFSET_REG
>         subl    %eax, %OFFSET_REG
>
>         vmovd   (%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %xmm0
> @@ -1386,7 +1570,7 @@ L(less_4_loop):
>  #  endif
>         incq    %rdi
>         /* end condition is reach page boundary (rdi is aligned).  */
> -       testl   $31, %edi
> +       testb   $(VEC_SIZE - 1), %dil
>         jnz     L(less_4_loop)
>         leaq    -(VEC_SIZE * 4)(%rdi, %rsi), %rsi
>         addq    $-(VEC_SIZE * 4), %rdi
> --
> 2.34.1
>

LGTM.

Thanks.
diff mbox series

Patch

diff --git a/sysdeps/x86_64/multiarch/strcmp-evex.S b/sysdeps/x86_64/multiarch/strcmp-evex.S
index e482d0167f..756a3bb8d6 100644
--- a/sysdeps/x86_64/multiarch/strcmp-evex.S
+++ b/sysdeps/x86_64/multiarch/strcmp-evex.S
@@ -20,6 +20,10 @@ 
 
 #if ISA_SHOULD_BUILD (4)
 
+# ifndef VEC_SIZE
+#  include "x86-evex256-vecs.h"
+# endif
+
 # define STRCMP_ISA	_evex
 # include "strcmp-naming.h"
 
@@ -35,41 +39,57 @@ 
 # define PAGE_SIZE	4096
 
 	/* VEC_SIZE = Number of bytes in a ymm register.  */
-# define VEC_SIZE	32
 # define CHAR_PER_VEC	(VEC_SIZE	/	SIZE_OF_CHAR)
 
-# define VMOVU	vmovdqu64
-# define VMOVA	vmovdqa64
-
 # ifdef USE_AS_WCSCMP
-#  define TESTEQ	subl $0xff,
 	/* Compare packed dwords.  */
 #  define VPCMP	vpcmpd
+#  define VPCMPEQ	vpcmpeqd
 #  define VPMINU	vpminud
 #  define VPTESTM	vptestmd
 #  define VPTESTNM	vptestnmd
 	/* 1 dword char == 4 bytes.  */
 #  define SIZE_OF_CHAR	4
+
+#  define TESTEQ	sub $((1 << CHAR_PER_VEC) - 1),
+
+#  define USE_WIDE_CHAR
 # else
-#  define TESTEQ	incl
 	/* Compare packed bytes.  */
 #  define VPCMP	vpcmpb
+#  define VPCMPEQ	vpcmpeqb
 #  define VPMINU	vpminub
 #  define VPTESTM	vptestmb
 #  define VPTESTNM	vptestnmb
 	/* 1 byte char == 1 byte.  */
 #  define SIZE_OF_CHAR	1
+
+#  define TESTEQ	inc
+# endif
+
+# include "reg-macros.h"
+
+# if VEC_SIZE == 64
+#  define RODATA_SECTION	rodata.cst64
+# else
+#  define RODATA_SECTION	rodata.cst32
+# endif
+
+# if CHAR_PER_VEC == 64
+#  define FALLTHROUGH_RETURN_OFFSET	(VEC_SIZE * 3)
+# else
+#  define FALLTHROUGH_RETURN_OFFSET	(VEC_SIZE * 2)
 # endif
 
 # ifdef USE_AS_STRNCMP
-#  define LOOP_REG	r9d
+#  define LOOP_REG	VR9
 #  define LOOP_REG64	r9
 
 #  define OFFSET_REG8	r9b
 #  define OFFSET_REG	r9d
 #  define OFFSET_REG64	r9
 # else
-#  define LOOP_REG	edx
+#  define LOOP_REG	VRDX
 #  define LOOP_REG64	rdx
 
 #  define OFFSET_REG8	dl
@@ -83,32 +103,6 @@ 
 #  define VEC_OFFSET	(-VEC_SIZE)
 # endif
 
-# define XMM0	xmm17
-# define XMM1	xmm18
-
-# define XMM10	xmm27
-# define XMM11	xmm28
-# define XMM12	xmm29
-# define XMM13	xmm30
-# define XMM14	xmm31
-
-
-# define YMM0	ymm17
-# define YMM1	ymm18
-# define YMM2	ymm19
-# define YMM3	ymm20
-# define YMM4	ymm21
-# define YMM5	ymm22
-# define YMM6	ymm23
-# define YMM7	ymm24
-# define YMM8	ymm25
-# define YMM9	ymm26
-# define YMM10	ymm27
-# define YMM11	ymm28
-# define YMM12	ymm29
-# define YMM13	ymm30
-# define YMM14	ymm31
-
 # ifdef USE_AS_STRCASECMP_L
 #  define BYTE_LOOP_REG	OFFSET_REG
 # else
@@ -125,61 +119,72 @@ 
 #  endif
 # endif
 
-# define LCASE_MIN_YMM	%YMM12
-# define LCASE_MAX_YMM	%YMM13
-# define CASE_ADD_YMM	%YMM14
+# define LCASE_MIN_V	VMM(12)
+# define LCASE_MAX_V	VMM(13)
+# define CASE_ADD_V	VMM(14)
 
-# define LCASE_MIN_XMM	%XMM12
-# define LCASE_MAX_XMM	%XMM13
-# define CASE_ADD_XMM	%XMM14
+# if VEC_SIZE == 64
+#  define LCASE_MIN_YMM	VMM_256(12)
+#  define LCASE_MAX_YMM	VMM_256(13)
+#  define CASE_ADD_YMM	VMM_256(14)
+# endif
+
+# define LCASE_MIN_XMM	VMM_128(12)
+# define LCASE_MAX_XMM	VMM_128(13)
+# define CASE_ADD_XMM	VMM_128(14)
 
 	/* NB: wcsncmp uses r11 but strcasecmp is never used in
 	   conjunction with wcscmp.  */
 # define TOLOWER_BASE	%r11
 
 # ifdef USE_AS_STRCASECMP_L
-#  define _REG(x, y) x ## y
-#  define REG(x, y) _REG(x, y)
-#  define TOLOWER(reg1, reg2, ext)										\
-	vpsubb	REG(LCASE_MIN_, ext), reg1, REG(%ext, 10);					\
-	vpsubb	REG(LCASE_MIN_, ext), reg2, REG(%ext, 11);					\
-	vpcmpub	$1, REG(LCASE_MAX_, ext), REG(%ext, 10), %k5;				\
-	vpcmpub	$1, REG(LCASE_MAX_, ext), REG(%ext, 11), %k6;				\
-	vpaddb	reg1, REG(CASE_ADD_, ext), reg1{%k5};						\
-	vpaddb	reg2, REG(CASE_ADD_, ext), reg2{%k6}
-
-#  define TOLOWER_gpr(src, dst) movl (TOLOWER_BASE, src, 4), dst
-#  define TOLOWER_YMM(...)	TOLOWER(__VA_ARGS__, YMM)
-#  define TOLOWER_XMM(...)	TOLOWER(__VA_ARGS__, XMM)
-
-#  define CMP_R1_R2(s1_reg, s2_reg, reg_out, ext)						\
-	TOLOWER	(s1_reg, s2_reg, ext);										\
-	VPCMP	$0, s1_reg, s2_reg, reg_out
-
-#  define CMP_R1_S2(s1_reg, s2_mem, s2_reg, reg_out, ext)				\
-	VMOVU	s2_mem, s2_reg;												\
-	CMP_R1_R2(s1_reg, s2_reg, reg_out, ext)
-
-#  define CMP_R1_R2_YMM(...) CMP_R1_R2(__VA_ARGS__, YMM)
-#  define CMP_R1_R2_XMM(...) CMP_R1_R2(__VA_ARGS__, XMM)
-
-#  define CMP_R1_S2_YMM(...) CMP_R1_S2(__VA_ARGS__, YMM)
-#  define CMP_R1_S2_XMM(...) CMP_R1_S2(__VA_ARGS__, XMM)
+#  define _REG(x, y)	x ## y
+#  define REG(x, y)	_REG(x, y)
+#  define TOLOWER(reg1, reg2, ext, vec_macro)	\
+	vpsubb	%REG(LCASE_MIN_, ext), reg1, %vec_macro(10);	\
+	vpsubb	%REG(LCASE_MIN_, ext), reg2, %vec_macro(11);	\
+	vpcmpub	$1, %REG(LCASE_MAX_, ext), %vec_macro(10), %k5;	\
+	vpcmpub	$1, %REG(LCASE_MAX_, ext), %vec_macro(11), %k6;	\
+	vpaddb	reg1, %REG(CASE_ADD_, ext), reg1{%k5};	\
+	vpaddb	reg2, %REG(CASE_ADD_, ext), reg2{%k6}
+
+#  define TOLOWER_gpr(src, dst)	movl (TOLOWER_BASE, src, 4), dst
+#  define TOLOWER_VMM(...)	TOLOWER(__VA_ARGS__, V, VMM)
+#  define TOLOWER_YMM(...)	TOLOWER(__VA_ARGS__, YMM, VMM_256)
+#  define TOLOWER_XMM(...)	TOLOWER(__VA_ARGS__, XMM, VMM_128)
+
+#  define CMP_R1_R2(s1_reg, s2_reg, reg_out, ext, vec_macro)	\
+	TOLOWER	(s1_reg, s2_reg, ext, vec_macro);	\
+	VPCMPEQ	s1_reg, s2_reg, reg_out
+
+#  define CMP_R1_S2(s1_reg, s2_mem, s2_reg, reg_out, ext, vec_macro)	\
+	VMOVU	s2_mem, s2_reg;	\
+	CMP_R1_R2 (s1_reg, s2_reg, reg_out, ext, vec_macro)
+
+#  define CMP_R1_R2_VMM(...)	CMP_R1_R2(__VA_ARGS__, V, VMM)
+#  define CMP_R1_R2_YMM(...)	CMP_R1_R2(__VA_ARGS__, YMM, VMM_256)
+#  define CMP_R1_R2_XMM(...)	CMP_R1_R2(__VA_ARGS__, XMM, VMM_128)
+
+#  define CMP_R1_S2_VMM(...)	CMP_R1_S2(__VA_ARGS__, V, VMM)
+#  define CMP_R1_S2_YMM(...)	CMP_R1_S2(__VA_ARGS__, YMM, VMM_256)
+#  define CMP_R1_S2_XMM(...)	CMP_R1_S2(__VA_ARGS__, XMM, VMM_128)
 
 # else
 #  define TOLOWER_gpr(...)
+#  define TOLOWER_VMM(...)
 #  define TOLOWER_YMM(...)
 #  define TOLOWER_XMM(...)
 
-#  define CMP_R1_R2_YMM(s1_reg, s2_reg, reg_out)						\
-	VPCMP	$0, s2_reg, s1_reg, reg_out
+#  define CMP_R1_R2_VMM(s1_reg, s2_reg, reg_out)	\
+	VPCMPEQ	s2_reg, s1_reg, reg_out
 
-#  define CMP_R1_R2_XMM(...) CMP_R1_R2_YMM(__VA_ARGS__)
+#  define CMP_R1_R2_YMM(...)	CMP_R1_R2_VMM(__VA_ARGS__)
+#  define CMP_R1_R2_XMM(...)	CMP_R1_R2_VMM(__VA_ARGS__)
 
-#  define CMP_R1_S2_YMM(s1_reg, s2_mem, unused, reg_out)				\
-	VPCMP	$0, s2_mem, s1_reg, reg_out
-
-#  define CMP_R1_S2_XMM(...) CMP_R1_S2_YMM(__VA_ARGS__)
+#  define CMP_R1_S2_VMM(s1_reg, s2_mem, unused, reg_out)	\
+	VPCMPEQ	s2_mem, s1_reg, reg_out
+#  define CMP_R1_S2_YMM(...)	CMP_R1_S2_VMM(__VA_ARGS__)
+#  define CMP_R1_S2_XMM(...)	CMP_R1_S2_VMM(__VA_ARGS__)
 # endif
 
 /* Warning!
@@ -203,7 +208,7 @@ 
    the maximum offset is reached before a difference is found, zero is
    returned.  */
 
-	.section .text.evex, "ax", @progbits
+	.section SECTION(.text), "ax", @progbits
 	.align	16
 	.type	STRCMP, @function
 	.globl	STRCMP
@@ -232,7 +237,7 @@  STRCMP:
 #  else
 	mov	(%LOCALE_REG), %RAX_LP
 #  endif
-	testl	$1, LOCALE_DATA_VALUES + _NL_CTYPE_NONASCII_CASE * SIZEOF_VALUES(%rax)
+	testb	$1, LOCALE_DATA_VALUES + _NL_CTYPE_NONASCII_CASE * SIZEOF_VALUES(%rax)
 	jne	STRCASECMP_L_NONASCII
 	leaq	_nl_C_LC_CTYPE_tolower + 128 * 4(%rip), TOLOWER_BASE
 # endif
@@ -254,28 +259,46 @@  STRCMP:
 # endif
 
 # if defined USE_AS_STRCASECMP_L
-	.section .rodata.cst32, "aM", @progbits, 32
-	.align	32
+	.section RODATA_SECTION, "aM", @progbits, VEC_SIZE
+	.align	VEC_SIZE
 L(lcase_min):
 	.quad	0x4141414141414141
 	.quad	0x4141414141414141
 	.quad	0x4141414141414141
 	.quad	0x4141414141414141
+#  if VEC_SIZE == 64
+	.quad	0x4141414141414141
+	.quad	0x4141414141414141
+	.quad	0x4141414141414141
+	.quad	0x4141414141414141
+#  endif
 L(lcase_max):
 	.quad	0x1a1a1a1a1a1a1a1a
 	.quad	0x1a1a1a1a1a1a1a1a
 	.quad	0x1a1a1a1a1a1a1a1a
 	.quad	0x1a1a1a1a1a1a1a1a
+#  if VEC_SIZE == 64
+	.quad	0x1a1a1a1a1a1a1a1a
+	.quad	0x1a1a1a1a1a1a1a1a
+	.quad	0x1a1a1a1a1a1a1a1a
+	.quad	0x1a1a1a1a1a1a1a1a
+#  endif
 L(case_add):
 	.quad	0x2020202020202020
 	.quad	0x2020202020202020
 	.quad	0x2020202020202020
 	.quad	0x2020202020202020
+#  if VEC_SIZE == 64
+	.quad	0x2020202020202020
+	.quad	0x2020202020202020
+	.quad	0x2020202020202020
+	.quad	0x2020202020202020
+#  endif
 	.previous
 
-	vmovdqa64 L(lcase_min)(%rip), LCASE_MIN_YMM
-	vmovdqa64 L(lcase_max)(%rip), LCASE_MAX_YMM
-	vmovdqa64 L(case_add)(%rip), CASE_ADD_YMM
+	VMOVA	L(lcase_min)(%rip), %LCASE_MIN_V
+	VMOVA	L(lcase_max)(%rip), %LCASE_MAX_V
+	VMOVA	L(case_add)(%rip), %CASE_ADD_V
 # endif
 
 	movl	%edi, %eax
@@ -288,12 +311,12 @@  L(case_add):
 
 L(no_page_cross):
 	/* Safe to compare 4x vectors.  */
-	VMOVU	(%rdi), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
+	VMOVU	(%rdi), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
 	/* Each bit cleared in K1 represents a mismatch or a null CHAR
 	   in YMM0 and 32 bytes at (%rsi).  */
-	CMP_R1_S2_YMM (%YMM0, (%rsi), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
+	CMP_R1_S2_VMM (%VMM(0), (%rsi), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
 # ifdef USE_AS_STRNCMP
 	cmpq	$CHAR_PER_VEC, %rdx
 	jbe	L(vec_0_test_len)
@@ -303,14 +326,14 @@  L(no_page_cross):
 	   wcscmp/wcsncmp.  */
 
 	/* All 1s represents all equals. TESTEQ will overflow to zero in
-	   all equals case. Otherwise 1s will carry until position of first
-	   mismatch.  */
-	TESTEQ	%ecx
+	   all equals case. Otherwise 1s will carry until position of
+	   first mismatch.  */
+	TESTEQ	%VRCX
 	jz	L(more_3x_vec)
 
 	.p2align 4,, 4
 L(return_vec_0):
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 # ifdef USE_AS_WCSCMP
 	movl	(%rdi, %rcx, SIZE_OF_CHAR), %edx
 	xorl	%eax, %eax
@@ -321,7 +344,16 @@  L(return_vec_0):
 	orl	$1, %eax
 # else
 	movzbl	(%rdi, %rcx), %eax
+	/* For VEC_SIZE == 64 use movb instead of movzbl to save a byte
+	   and keep logic for len <= VEC_SIZE (common) in just the
+	   first cache line.  NB: No evex512 processor has partial-
+	   register stalls. If that changes this ifdef can be disabled
+	   without affecting correctness.  */
+#  if !defined USE_AS_STRNCMP && !defined USE_AS_STRCASECMP_L && VEC_SIZE == 64
+	movb	(%rsi, %rcx), %cl
+#  else
 	movzbl	(%rsi, %rcx), %ecx
+#  endif
 	TOLOWER_gpr (%rax, %eax)
 	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
@@ -332,8 +364,8 @@  L(ret0):
 # ifdef USE_AS_STRNCMP
 	.p2align 4,, 4
 L(vec_0_test_len):
-	notl	%ecx
-	bzhil	%edx, %ecx, %eax
+	not	%VRCX
+	bzhi	%VRDX, %VRCX, %VRAX
 	jnz	L(return_vec_0)
 	/* Align if will cross fetch block.  */
 	.p2align 4,, 2
@@ -372,7 +404,7 @@  L(ret1):
 
 	.p2align 4,, 10
 L(return_vec_1):
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 # ifdef USE_AS_STRNCMP
 	/* rdx must be > CHAR_PER_VEC so its safe to subtract without
 	   worrying about underflow.  */
@@ -401,24 +433,41 @@  L(ret2):
 	.p2align 4,, 10
 # ifdef USE_AS_STRNCMP
 L(return_vec_3):
-#  if CHAR_PER_VEC <= 16
+#  if CHAR_PER_VEC <= 32
+	/* If CHAR_PER_VEC <= 32 reuse code from L(return_vec_3) without
+	   additional branches by adjusting the bit positions from
+	   VEC3.  We can't do this for CHAR_PER_VEC == 64.  */
+#   if CHAR_PER_VEC <= 16
 	sall	$CHAR_PER_VEC, %ecx
-#  else
+#   else
 	salq	$CHAR_PER_VEC, %rcx
+#   endif
+#  else
+	/* If CHAR_PER_VEC == 64 we can't shift the return GPR so just
+	   check it.  */
+	bsf	%VRCX, %VRCX
+	addl	$(CHAR_PER_VEC), %ecx
+	cmpq	%rcx, %rdx
+	ja	L(ret_vec_3_finish)
+	xorl	%eax, %eax
+	ret
 #  endif
 # endif
+
+	/* If CHAR_PER_VEC == 64 we can't combine matches from the last
+	   2x VEC so need seperate return label.  */
 L(return_vec_2):
 # if (CHAR_PER_VEC <= 16) || !(defined USE_AS_STRNCMP)
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 # else
-	tzcntq	%rcx, %rcx
+	bsfq	%rcx, %rcx
 # endif
-
 # ifdef USE_AS_STRNCMP
 	cmpq	%rcx, %rdx
 	jbe	L(ret_zero)
 # endif
 
+L(ret_vec_3_finish):
 # ifdef USE_AS_WCSCMP
 	movl	(VEC_SIZE * 2)(%rdi, %rcx, SIZE_OF_CHAR), %edx
 	xorl	%eax, %eax
@@ -440,7 +489,7 @@  L(ret3):
 # ifndef USE_AS_STRNCMP
 	.p2align 4,, 10
 L(return_vec_3):
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 #  ifdef USE_AS_WCSCMP
 	movl	(VEC_SIZE * 3)(%rdi, %rcx, SIZE_OF_CHAR), %edx
 	xorl	%eax, %eax
@@ -465,11 +514,11 @@  L(ret4):
 	.p2align 5
 L(more_3x_vec):
 	/* Safe to compare 4x vectors.  */
-	VMOVU	(VEC_SIZE)(%rdi), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, VEC_SIZE(%rsi), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVU	(VEC_SIZE)(%rdi), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), VEC_SIZE(%rsi), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_1)
 
 # ifdef USE_AS_STRNCMP
@@ -477,18 +526,18 @@  L(more_3x_vec):
 	jbe	L(ret_zero)
 # endif
 
-	VMOVU	(VEC_SIZE * 2)(%rdi), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, (VEC_SIZE * 2)(%rsi), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVU	(VEC_SIZE * 2)(%rdi), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), (VEC_SIZE * 2)(%rsi), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_2)
 
-	VMOVU	(VEC_SIZE * 3)(%rdi), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, (VEC_SIZE * 3)(%rsi), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVU	(VEC_SIZE * 3)(%rdi), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), (VEC_SIZE * 3)(%rsi), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_3)
 
 # ifdef USE_AS_STRNCMP
@@ -565,110 +614,123 @@  L(loop):
 
 	/* Loop entry after handling page cross during loop.  */
 L(loop_skip_page_cross_check):
-	VMOVA	(VEC_SIZE * 0)(%rdi), %YMM0
-	VMOVA	(VEC_SIZE * 1)(%rdi), %YMM2
-	VMOVA	(VEC_SIZE * 2)(%rdi), %YMM4
-	VMOVA	(VEC_SIZE * 3)(%rdi), %YMM6
+	VMOVA	(VEC_SIZE * 0)(%rdi), %VMM(0)
+	VMOVA	(VEC_SIZE * 1)(%rdi), %VMM(2)
+	VMOVA	(VEC_SIZE * 2)(%rdi), %VMM(4)
+	VMOVA	(VEC_SIZE * 3)(%rdi), %VMM(6)
 
-	VPMINU	%YMM0, %YMM2, %YMM8
-	VPMINU	%YMM4, %YMM6, %YMM9
+	VPMINU	%VMM(0), %VMM(2), %VMM(8)
+	VPMINU	%VMM(4), %VMM(6), %VMM(9)
 
 	/* A zero CHAR in YMM9 means that there is a null CHAR.  */
-	VPMINU	%YMM8, %YMM9, %YMM9
+	VPMINU	%VMM(8), %VMM(9), %VMM(9)
 
 	/* Each bit set in K1 represents a non-null CHAR in YMM9.  */
-	VPTESTM	%YMM9, %YMM9, %k1
+	VPTESTM	%VMM(9), %VMM(9), %k1
 # ifndef USE_AS_STRCASECMP_L
-	vpxorq	(VEC_SIZE * 0)(%rsi), %YMM0, %YMM1
-	vpxorq	(VEC_SIZE * 1)(%rsi), %YMM2, %YMM3
-	vpxorq	(VEC_SIZE * 2)(%rsi), %YMM4, %YMM5
+	vpxorq	(VEC_SIZE * 0)(%rsi), %VMM(0), %VMM(1)
+	vpxorq	(VEC_SIZE * 1)(%rsi), %VMM(2), %VMM(3)
+	vpxorq	(VEC_SIZE * 2)(%rsi), %VMM(4), %VMM(5)
 	/* Ternary logic to xor (VEC_SIZE * 3)(%rsi) with YMM6 while
 	   oring with YMM1. Result is stored in YMM6.  */
-	vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %YMM1, %YMM6
+	vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %VMM(1), %VMM(6)
 # else
-	VMOVU	(VEC_SIZE * 0)(%rsi), %YMM1
-	TOLOWER_YMM (%YMM0, %YMM1)
-	VMOVU	(VEC_SIZE * 1)(%rsi), %YMM3
-	TOLOWER_YMM (%YMM2, %YMM3)
-	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM5
-	TOLOWER_YMM (%YMM4, %YMM5)
-	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM7
-	TOLOWER_YMM (%YMM6, %YMM7)
-	vpxorq	%YMM0, %YMM1, %YMM1
-	vpxorq	%YMM2, %YMM3, %YMM3
-	vpxorq	%YMM4, %YMM5, %YMM5
-	vpternlogd $0xde, %YMM7, %YMM1, %YMM6
+	VMOVU	(VEC_SIZE * 0)(%rsi), %VMM(1)
+	TOLOWER_VMM (%VMM(0), %VMM(1))
+	VMOVU	(VEC_SIZE * 1)(%rsi), %VMM(3)
+	TOLOWER_VMM (%VMM(2), %VMM(3))
+	VMOVU	(VEC_SIZE * 2)(%rsi), %VMM(5)
+	TOLOWER_VMM (%VMM(4), %VMM(5))
+	VMOVU	(VEC_SIZE * 3)(%rsi), %VMM(7)
+	TOLOWER_VMM (%VMM(6), %VMM(7))
+	vpxorq	%VMM(0), %VMM(1), %VMM(1)
+	vpxorq	%VMM(2), %VMM(3), %VMM(3)
+	vpxorq	%VMM(4), %VMM(5), %VMM(5)
+	vpternlogd $0xde, %VMM(7), %VMM(1), %VMM(6)
 # endif
 	/* Or together YMM3, YMM5, and YMM6.  */
-	vpternlogd $0xfe, %YMM3, %YMM5, %YMM6
+	vpternlogd $0xfe, %VMM(3), %VMM(5), %VMM(6)
 
 
 	/* A non-zero CHAR in YMM6 represents a mismatch.  */
-	VPTESTNM %YMM6, %YMM6, %k0{%k1}
-	kmovd	%k0, %LOOP_REG
+	VPTESTNM %VMM(6), %VMM(6), %k0{%k1}
+	KMOV	%k0, %LOOP_REG
 
 	TESTEQ	%LOOP_REG
 	jz	L(loop)
 
 
 	/* Find which VEC has the mismatch of end of string.  */
-	VPTESTM	%YMM0, %YMM0, %k1
-	VPTESTNM %YMM1, %YMM1, %k0{%k1}
-	kmovd	%k0, %ecx
-	TESTEQ	%ecx
+	VPTESTM	%VMM(0), %VMM(0), %k1
+	VPTESTNM %VMM(1), %VMM(1), %k0{%k1}
+	KMOV	%k0, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_0_end)
 
-	VPTESTM	%YMM2, %YMM2, %k1
-	VPTESTNM %YMM3, %YMM3, %k0{%k1}
-	kmovd	%k0, %ecx
-	TESTEQ	%ecx
+	VPTESTM	%VMM(2), %VMM(2), %k1
+	VPTESTNM %VMM(3), %VMM(3), %k0{%k1}
+	KMOV	%k0, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_1_end)
 
 
-	/* Handle VEC 2 and 3 without branches.  */
+	/* Handle VEC 2 and 3 without branches if CHAR_PER_VEC <= 32.
+	 */
 L(return_vec_2_3_end):
 # ifdef USE_AS_STRNCMP
 	subq	$(CHAR_PER_VEC * 2), %rdx
 	jbe	L(ret_zero_end)
 # endif
 
-	VPTESTM	%YMM4, %YMM4, %k1
-	VPTESTNM %YMM5, %YMM5, %k0{%k1}
-	kmovd	%k0, %ecx
-	TESTEQ	%ecx
+	VPTESTM	%VMM(4), %VMM(4), %k1
+	VPTESTNM %VMM(5), %VMM(5), %k0{%k1}
+	KMOV	%k0, %VRCX
+	TESTEQ	%VRCX
 # if CHAR_PER_VEC <= 16
 	sall	$CHAR_PER_VEC, %LOOP_REG
 	orl	%ecx, %LOOP_REG
-# else
+# elif CHAR_PER_VEC <= 32
 	salq	$CHAR_PER_VEC, %LOOP_REG64
 	orq	%rcx, %LOOP_REG64
+# else
+	/* We aren't combining last 2x VEC so branch on second the last.
+	 */
+	jnz	L(return_vec_2_end)
 # endif
-L(return_vec_3_end):
+
 	/* LOOP_REG contains matches for null/mismatch from the loop. If
-	   VEC 0,1,and 2 all have no null and no mismatches then mismatch
-	   must entirely be from VEC 3 which is fully represented by
-	   LOOP_REG.  */
+	   VEC 0,1,and 2 all have no null and no mismatches then
+	   mismatch must entirely be from VEC 3 which is fully
+	   represented by LOOP_REG.  */
 # if CHAR_PER_VEC <= 16
-	tzcntl	%LOOP_REG, %LOOP_REG
+	bsf	%LOOP_REG, %LOOP_REG
 # else
-	tzcntq	%LOOP_REG64, %LOOP_REG64
+	bsfq	%LOOP_REG64, %LOOP_REG64
 # endif
 # ifdef USE_AS_STRNCMP
+
+	/* If CHAR_PER_VEC == 64 we can't combine last 2x VEC so need to
+	   adj length before last comparison.  */
+#  if CHAR_PER_VEC == 64
+	subq	$CHAR_PER_VEC, %rdx
+	jbe	L(ret_zero_end)
+#  endif
+
 	cmpq	%LOOP_REG64, %rdx
 	jbe	L(ret_zero_end)
 # endif
 
 # ifdef USE_AS_WCSCMP
-	movl	(VEC_SIZE * 2)(%rdi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
+	movl	(FALLTHROUGH_RETURN_OFFSET)(%rdi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
 	xorl	%eax, %eax
-	cmpl	(VEC_SIZE * 2)(%rsi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
+	cmpl	(FALLTHROUGH_RETURN_OFFSET)(%rsi, %LOOP_REG64, SIZE_OF_CHAR), %ecx
 	je	L(ret5)
 	setl	%al
 	negl	%eax
 	xorl	%r8d, %eax
 # else
-	movzbl	(VEC_SIZE * 2)(%rdi, %LOOP_REG64), %eax
-	movzbl	(VEC_SIZE * 2)(%rsi, %LOOP_REG64), %ecx
+	movzbl	(FALLTHROUGH_RETURN_OFFSET)(%rdi, %LOOP_REG64), %eax
+	movzbl	(FALLTHROUGH_RETURN_OFFSET)(%rsi, %LOOP_REG64), %ecx
 	TOLOWER_gpr (%rax, %eax)
 	TOLOWER_gpr (%rcx, %ecx)
 	subl	%ecx, %eax
@@ -686,23 +748,39 @@  L(ret_zero_end):
 # endif
 
 
+
 	/* The L(return_vec_N_end) differ from L(return_vec_N) in that
-	   they use the value of `r8` to negate the return value. This is
-	   because the page cross logic can swap `rdi` and `rsi`.  */
+	   they use the value of `r8` to negate the return value. This
+	   is because the page cross logic can swap `rdi` and `rsi`.
+	 */
 	.p2align 4,, 10
 # ifdef USE_AS_STRNCMP
 L(return_vec_1_end):
-#  if CHAR_PER_VEC <= 16
+#  if CHAR_PER_VEC <= 32
+	/* If CHAR_PER_VEC <= 32 reuse code from L(return_vec_0_end)
+	   without additional branches by adjusting the bit positions
+	   from VEC1.  We can't do this for CHAR_PER_VEC == 64.  */
+#   if CHAR_PER_VEC <= 16
 	sall	$CHAR_PER_VEC, %ecx
-#  else
+#   else
 	salq	$CHAR_PER_VEC, %rcx
+#   endif
+#  else
+	/* If CHAR_PER_VEC == 64 we can't shift the return GPR so just
+	   check it.  */
+	bsf	%VRCX, %VRCX
+	addl	$(CHAR_PER_VEC), %ecx
+	cmpq	%rcx, %rdx
+	ja	L(ret_vec_0_end_finish)
+	xorl	%eax, %eax
+	ret
 #  endif
 # endif
 L(return_vec_0_end):
 # if (CHAR_PER_VEC <= 16) || !(defined USE_AS_STRNCMP)
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 # else
-	tzcntq	%rcx, %rcx
+	bsfq	%rcx, %rcx
 # endif
 
 # ifdef USE_AS_STRNCMP
@@ -710,6 +788,7 @@  L(return_vec_0_end):
 	jbe	L(ret_zero_end)
 # endif
 
+L(ret_vec_0_end_finish):
 # ifdef USE_AS_WCSCMP
 	movl	(%rdi, %rcx, SIZE_OF_CHAR), %edx
 	xorl	%eax, %eax
@@ -737,7 +816,7 @@  L(ret6):
 # ifndef USE_AS_STRNCMP
 	.p2align 4,, 10
 L(return_vec_1_end):
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 #  ifdef USE_AS_WCSCMP
 	movl	VEC_SIZE(%rdi, %rcx, SIZE_OF_CHAR), %edx
 	xorl	%eax, %eax
@@ -760,6 +839,41 @@  L(ret7):
 # endif
 
 
+	/* If CHAR_PER_VEC == 64 we can't combine matches from the last
+	   2x VEC so need seperate return label.  */
+# if CHAR_PER_VEC == 64
+L(return_vec_2_end):
+	bsf	%VRCX, %VRCX
+#  ifdef USE_AS_STRNCMP
+	cmpq	%rcx, %rdx
+	jbe	L(ret_zero_end)
+#  endif
+#  ifdef USE_AS_WCSCMP
+	movl	(VEC_SIZE * 2)(%rdi, %rcx, SIZE_OF_CHAR), %edx
+	xorl	%eax, %eax
+	cmpl	(VEC_SIZE * 2)(%rsi, %rcx, SIZE_OF_CHAR), %edx
+	je	L(ret31)
+	setl	%al
+	negl	%eax
+	/* This is the non-zero case for `eax` so just xorl with `r8d`
+	   flip is `rdi` and `rsi` where swapped.  */
+	xorl	%r8d, %eax
+#  else
+	movzbl	(VEC_SIZE * 2)(%rdi, %rcx), %eax
+	movzbl	(VEC_SIZE * 2)(%rsi, %rcx), %ecx
+	TOLOWER_gpr (%rax, %eax)
+	TOLOWER_gpr (%rcx, %ecx)
+	subl	%ecx, %eax
+	/* Flip `eax` if `rdi` and `rsi` where swapped in page cross
+	   logic. Subtract `r8d` after xor for zero case.  */
+	xorl	%r8d, %eax
+	subl	%r8d, %eax
+#  endif
+L(ret13):
+	ret
+# endif
+
+
 	/* Page cross in rsi in next 4x VEC.  */
 
 	/* TODO: Improve logic here.  */
@@ -778,11 +892,11 @@  L(page_cross_during_loop):
 	cmpl	$-(VEC_SIZE * 3), %eax
 	jle	L(less_1x_vec_till_page_cross)
 
-	VMOVA	(%rdi), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, (%rsi), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVA	(%rdi), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), (%rsi), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_0_end)
 
 	/* if distance >= 2x VEC then eax > -(VEC_SIZE * 2).  */
@@ -799,9 +913,9 @@  L(less_1x_vec_till_page_cross):
 	   to read back -VEC_SIZE. If rdi is truly at the start of a page
 	   here, it means the previous page (rdi - VEC_SIZE) has already
 	   been loaded earlier so must be valid.  */
-	VMOVU	-VEC_SIZE(%rdi, %rax), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, -VEC_SIZE(%rsi, %rax), %YMM1, %k1){%k2}
+	VMOVU	-VEC_SIZE(%rdi, %rax), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), -VEC_SIZE(%rsi, %rax), %VMM(1), %k1){%k2}
 	/* Mask of potentially valid bits. The lower bits can be out of
 	   range comparisons (but safe regarding page crosses).  */
 
@@ -813,12 +927,12 @@  L(less_1x_vec_till_page_cross):
 	shlxl	%ecx, %r10d, %ecx
 	movzbl	%cl, %r10d
 # else
-	movl	$-1, %ecx
-	shlxl	%esi, %ecx, %r10d
+	mov	$-1, %VRCX
+	shlx	%VRSI, %VRCX, %VR10
 # endif
 
-	kmovd	%k1, %ecx
-	notl	%ecx
+	KMOV	%k1, %VRCX
+	not	%VRCX
 
 
 # ifdef USE_AS_STRNCMP
@@ -838,12 +952,10 @@  L(less_1x_vec_till_page_cross):
 	/* Readjust eax before potentially returning to the loop.  */
 	addl	$(PAGE_SIZE - VEC_SIZE * 4), %eax
 
-	andl	%r10d, %ecx
+	and	%VR10, %VRCX
 	jz	L(loop_skip_page_cross_check)
 
-	.p2align 4,, 3
-L(return_page_cross_end):
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 
 # if (defined USE_AS_STRNCMP) || (defined USE_AS_WCSCMP)
 	leal	-VEC_SIZE(%OFFSET_REG64, %rcx, SIZE_OF_CHAR), %ecx
@@ -874,8 +986,12 @@  L(ret8):
 # ifdef USE_AS_STRNCMP
 	.p2align 4,, 10
 L(return_page_cross_end_check):
-	andl	%r10d, %ecx
-	tzcntl	%ecx, %ecx
+	and	%VR10, %VRCX
+	/* Need to use tzcnt here as VRCX may be zero.  If VRCX is zero
+	   tzcnt(VRCX) will be CHAR_PER and remaining length (edx) is
+	   guranteed to be <= CHAR_PER_VEC so we will only use the return
+	   idx if VRCX was non-zero.  */
+	tzcnt	%VRCX, %VRCX
 	leal	-VEC_SIZE(%rax, %rcx, SIZE_OF_CHAR), %ecx
 #  ifdef USE_AS_WCSCMP
 	sall	$2, %edx
@@ -892,11 +1008,11 @@  L(more_2x_vec_till_page_cross):
 	/* If more 2x vec till cross we will complete a full loop
 	   iteration here.  */
 
-	VMOVA	VEC_SIZE(%rdi), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, VEC_SIZE(%rsi), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVA	VEC_SIZE(%rdi), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), VEC_SIZE(%rsi), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_1_end)
 
 # ifdef USE_AS_STRNCMP
@@ -907,18 +1023,18 @@  L(more_2x_vec_till_page_cross):
 	subl	$-(VEC_SIZE * 4), %eax
 
 	/* Safe to include comparisons from lower bytes.  */
-	VMOVU	-(VEC_SIZE * 2)(%rdi, %rax), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, -(VEC_SIZE * 2)(%rsi, %rax), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVU	-(VEC_SIZE * 2)(%rdi, %rax), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), -(VEC_SIZE * 2)(%rsi, %rax), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_page_cross_0)
 
-	VMOVU	-(VEC_SIZE * 1)(%rdi, %rax), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, -(VEC_SIZE * 1)(%rsi, %rax), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVU	-(VEC_SIZE * 1)(%rdi, %rax), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), -(VEC_SIZE * 1)(%rsi, %rax), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(return_vec_page_cross_1)
 
 # ifdef USE_AS_STRNCMP
@@ -937,30 +1053,30 @@  L(more_2x_vec_till_page_cross):
 # endif
 
 	/* Finish the loop.  */
-	VMOVA	(VEC_SIZE * 2)(%rdi), %YMM4
-	VMOVA	(VEC_SIZE * 3)(%rdi), %YMM6
-	VPMINU	%YMM4, %YMM6, %YMM9
-	VPTESTM	%YMM9, %YMM9, %k1
+	VMOVA	(VEC_SIZE * 2)(%rdi), %VMM(4)
+	VMOVA	(VEC_SIZE * 3)(%rdi), %VMM(6)
+	VPMINU	%VMM(4), %VMM(6), %VMM(9)
+	VPTESTM	%VMM(9), %VMM(9), %k1
 # ifndef USE_AS_STRCASECMP_L
-	vpxorq	(VEC_SIZE * 2)(%rsi), %YMM4, %YMM5
+	vpxorq	(VEC_SIZE * 2)(%rsi), %VMM(4), %VMM(5)
 	/* YMM6 = YMM5 | ((VEC_SIZE * 3)(%rsi) ^ YMM6).  */
-	vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %YMM5, %YMM6
+	vpternlogd $0xde, (VEC_SIZE * 3)(%rsi), %VMM(5), %VMM(6)
 # else
-	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM5
-	TOLOWER_YMM (%YMM4, %YMM5)
-	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM7
-	TOLOWER_YMM (%YMM6, %YMM7)
-	vpxorq	%YMM4, %YMM5, %YMM5
-	vpternlogd $0xde, %YMM7, %YMM5, %YMM6
-# endif
-	VPTESTNM %YMM6, %YMM6, %k0{%k1}
-	kmovd	%k0, %LOOP_REG
+	VMOVU	(VEC_SIZE * 2)(%rsi), %VMM(5)
+	TOLOWER_VMM (%VMM(4), %VMM(5))
+	VMOVU	(VEC_SIZE * 3)(%rsi), %VMM(7)
+	TOLOWER_VMM (%VMM(6), %VMM(7))
+	vpxorq	%VMM(4), %VMM(5), %VMM(5)
+	vpternlogd $0xde, %VMM(7), %VMM(5), %VMM(6)
+# endif
+	VPTESTNM %VMM(6), %VMM(6), %k0{%k1}
+	KMOV	%k0, %LOOP_REG
 	TESTEQ	%LOOP_REG
 	jnz	L(return_vec_2_3_end)
 
 	/* Best for code size to include ucond-jmp here. Would be faster
-	   if this case is hot to duplicate the L(return_vec_2_3_end) code
-	   as fall-through and have jump back to loop on mismatch
+	   if this case is hot to duplicate the L(return_vec_2_3_end)
+	   code as fall-through and have jump back to loop on mismatch
 	   comparison.  */
 	subq	$-(VEC_SIZE * 4), %rdi
 	subq	$-(VEC_SIZE * 4), %rsi
@@ -980,7 +1096,7 @@  L(ret_zero_in_loop_page_cross):
 L(return_vec_page_cross_0):
 	addl	$-VEC_SIZE, %eax
 L(return_vec_page_cross_1):
-	tzcntl	%ecx, %ecx
+	bsf	%VRCX, %VRCX
 # if defined USE_AS_STRNCMP || defined USE_AS_WCSCMP
 	leal	-VEC_SIZE(%rax, %rcx, SIZE_OF_CHAR), %ecx
 #  ifdef USE_AS_STRNCMP
@@ -1023,8 +1139,8 @@  L(ret9):
 L(page_cross):
 # ifndef USE_AS_STRNCMP
 	/* If both are VEC aligned we don't need any special logic here.
-	   Only valid for strcmp where stop condition is guranteed to be
-	   reachable by just reading memory.  */
+	   Only valid for strcmp where stop condition is guranteed to
+	   be reachable by just reading memory.  */
 	testl	$((VEC_SIZE - 1) << 20), %eax
 	jz	L(no_page_cross)
 # endif
@@ -1065,11 +1181,11 @@  L(page_cross):
 	   loadable memory until within 1x VEC of page cross.  */
 	.p2align 4,, 8
 L(page_cross_loop):
-	VMOVU	(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM1, %k1){%k2}
-	kmovd	%k1, %ecx
-	TESTEQ	%ecx
+	VMOVU	(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(1), %k1){%k2}
+	KMOV	%k1, %VRCX
+	TESTEQ	%VRCX
 	jnz	L(check_ret_vec_page_cross)
 	addl	$CHAR_PER_VEC, %OFFSET_REG
 # ifdef USE_AS_STRNCMP
@@ -1087,13 +1203,13 @@  L(page_cross_loop):
 	subl	%eax, %OFFSET_REG
 	/* OFFSET_REG has distance to page cross - VEC_SIZE. Guranteed
 	   to not cross page so is safe to load. Since we have already
-	   loaded at least 1 VEC from rsi it is also guranteed to be safe.
-	 */
-	VMOVU	(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM0
-	VPTESTM	%YMM0, %YMM0, %k2
-	CMP_R1_S2_YMM (%YMM0, (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %YMM1, %k1){%k2}
+	   loaded at least 1 VEC from rsi it is also guranteed to be
+	   safe.  */
+	VMOVU	(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(0)
+	VPTESTM	%VMM(0), %VMM(0), %k2
+	CMP_R1_S2_VMM (%VMM(0), (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM(1), %k1){%k2}
 
-	kmovd	%k1, %ecx
+	KMOV	%k1, %VRCX
 # ifdef USE_AS_STRNCMP
 	leal	CHAR_PER_VEC(%OFFSET_REG64), %eax
 	cmpq	%rax, %rdx
@@ -1104,7 +1220,7 @@  L(page_cross_loop):
 	addq	%rdi, %rdx
 #  endif
 # endif
-	TESTEQ	%ecx
+	TESTEQ	%VRCX
 	jz	L(prepare_loop_no_len)
 
 	.p2align 4,, 4
@@ -1112,7 +1228,7 @@  L(ret_vec_page_cross):
 # ifndef USE_AS_STRNCMP
 L(check_ret_vec_page_cross):
 # endif
-	tzcntl	%ecx, %ecx
+	tzcnt	%VRCX, %VRCX
 	addl	%OFFSET_REG, %ecx
 L(ret_vec_page_cross_cont):
 # ifdef USE_AS_WCSCMP
@@ -1139,9 +1255,9 @@  L(ret12):
 # ifdef USE_AS_STRNCMP
 	.p2align 4,, 10
 L(check_ret_vec_page_cross2):
-	TESTEQ	%ecx
+	TESTEQ	%VRCX
 L(check_ret_vec_page_cross):
-	tzcntl	%ecx, %ecx
+	tzcnt	%VRCX, %VRCX
 	addl	%OFFSET_REG, %ecx
 	cmpq	%rcx, %rdx
 	ja	L(ret_vec_page_cross_cont)
@@ -1180,8 +1296,71 @@  L(less_1x_vec_till_page):
 # ifdef USE_AS_WCSCMP
 	shrl	$2, %eax
 # endif
+
+	/* Find largest load size we can use. VEC_SIZE == 64 only check
+	   if we can do a full ymm load.  */
+# if VEC_SIZE == 64
+
+	cmpl	$((VEC_SIZE - 32) / SIZE_OF_CHAR), %eax
+	ja	L(less_32_till_page)
+
+
+	/* Use 16 byte comparison.  */
+	VMOVU	(%rdi), %VMM_256(0)
+	VPTESTM	%VMM_256(0), %VMM_256(0), %k2
+	CMP_R1_S2_YMM (%VMM_256(0), (%rsi), %VMM_256(1), %k1){%k2}
+	kmovd	%k1, %ecx
+#  ifdef USE_AS_WCSCMP
+	subl	$0xff, %ecx
+#  else
+	incl	%ecx
+#  endif
+	jnz	L(check_ret_vec_page_cross)
+	movl	$((VEC_SIZE - 32) / SIZE_OF_CHAR), %OFFSET_REG
+#  ifdef USE_AS_STRNCMP
+	cmpq	%OFFSET_REG64, %rdx
+	jbe	L(ret_zero_page_cross_slow_case64)
+	subl	%eax, %OFFSET_REG
+#  else
+	/* Explicit check for 32 byte alignment.  */
+	subl	%eax, %OFFSET_REG
+	jz	L(prepare_loop)
+#  endif
+	VMOVU	(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM_256(0)
+	VPTESTM	%VMM_256(0), %VMM_256(0), %k2
+	CMP_R1_S2_YMM (%VMM_256(0), (%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %VMM_256(1), %k1){%k2}
+	kmovd	%k1, %ecx
+#  ifdef USE_AS_WCSCMP
+	subl	$0xff, %ecx
+#  else
+	incl	%ecx
+#  endif
+	jnz	L(check_ret_vec_page_cross)
+#  ifdef USE_AS_STRNCMP
+	addl	$(32 / SIZE_OF_CHAR), %OFFSET_REG
+	subq	%OFFSET_REG64, %rdx
+	jbe	L(ret_zero_page_cross_slow_case64)
+	subq	$-(CHAR_PER_VEC * 4), %rdx
+
+	leaq	-(VEC_SIZE * 4)(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %rdi
+	leaq	-(VEC_SIZE * 4)(%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %rsi
+#  else
+	leaq	(32 - VEC_SIZE * 4)(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %rdi
+	leaq	(32 - VEC_SIZE * 4)(%rsi, %OFFSET_REG64, SIZE_OF_CHAR), %rsi
+#  endif
+	jmp	L(prepare_loop_aligned)
+
+#  ifdef USE_AS_STRNCMP
+	.p2align 4,, 2
+L(ret_zero_page_cross_slow_case64):
+	xorl	%eax, %eax
+	ret
+#  endif
+L(less_32_till_page):
+# endif
+
 	/* Find largest load size we can use.  */
-	cmpl	$(16 / SIZE_OF_CHAR), %eax
+	cmpl	$((VEC_SIZE - 16) / SIZE_OF_CHAR), %eax
 	ja	L(less_16_till_page)
 
 	/* Use 16 byte comparison.  */
@@ -1195,9 +1374,14 @@  L(less_1x_vec_till_page):
 	incw	%cx
 # endif
 	jnz	L(check_ret_vec_page_cross)
-	movl	$(16 / SIZE_OF_CHAR), %OFFSET_REG
+
+	movl	$((VEC_SIZE - 16) / SIZE_OF_CHAR), %OFFSET_REG
 # ifdef USE_AS_STRNCMP
+#  if VEC_SIZE == 32
 	cmpq	%OFFSET_REG64, %rdx
+#  else
+	cmpq	$(16 / SIZE_OF_CHAR), %rdx
+#  endif
 	jbe	L(ret_zero_page_cross_slow_case0)
 	subl	%eax, %OFFSET_REG
 # else
@@ -1239,7 +1423,7 @@  L(ret_zero_page_cross_slow_case0):
 
 	.p2align 4,, 10
 L(less_16_till_page):
-	cmpl	$(24 / SIZE_OF_CHAR), %eax
+	cmpl	$((VEC_SIZE - 8) / SIZE_OF_CHAR), %eax
 	ja	L(less_8_till_page)
 
 	/* Use 8 byte comparison.  */
@@ -1260,7 +1444,7 @@  L(less_16_till_page):
 	cmpq	$(8 / SIZE_OF_CHAR), %rdx
 	jbe	L(ret_zero_page_cross_slow_case0)
 # endif
-	movl	$(24 / SIZE_OF_CHAR), %OFFSET_REG
+	movl	$((VEC_SIZE - 8) / SIZE_OF_CHAR), %OFFSET_REG
 	subl	%eax, %OFFSET_REG
 
 	vmovq	(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %xmm0
@@ -1320,7 +1504,7 @@  L(ret_less_8_wcs):
 	ret
 
 # else
-	cmpl	$28, %eax
+	cmpl	$(VEC_SIZE - 4), %eax
 	ja	L(less_4_till_page)
 
 	vmovd	(%rdi), %xmm0
@@ -1335,7 +1519,7 @@  L(ret_less_8_wcs):
 	cmpq	$4, %rdx
 	jbe	L(ret_zero_page_cross_slow_case1)
 #  endif
-	movl	$(28 / SIZE_OF_CHAR), %OFFSET_REG
+	movl	$((VEC_SIZE - 4) / SIZE_OF_CHAR), %OFFSET_REG
 	subl	%eax, %OFFSET_REG
 
 	vmovd	(%rdi, %OFFSET_REG64, SIZE_OF_CHAR), %xmm0
@@ -1386,7 +1570,7 @@  L(less_4_loop):
 #  endif
 	incq	%rdi
 	/* end condition is reach page boundary (rdi is aligned).  */
-	testl	$31, %edi
+	testb	$(VEC_SIZE - 1), %dil
 	jnz	L(less_4_loop)
 	leaq	-(VEC_SIZE * 4)(%rdi, %rsi), %rsi
 	addq	$-(VEC_SIZE * 4), %rdi