diff mbox series

[v1] x86: Move strrchr SSE2 implementation to multiarch/strrchr-sse2.S

Message ID 20220712192910.351121-4-goldstein.w.n@gmail.com
State New
Headers show
Series [v1] x86: Move strrchr SSE2 implementation to multiarch/strrchr-sse2.S | expand

Commit Message

Noah Goldstein July 12, 2022, 7:29 p.m. UTC
This commit doesn't affect libc.so.6, its just housekeeping to prepare
for adding explicit ISA level support.

Tested build on x86_64 and x86_32 with/without multiarch.
---
 sysdeps/x86_64/multiarch/strrchr-sse2.S | 358 ++++++++++++++++++++++-
 sysdeps/x86_64/multiarch/wcsrchr-sse2.S |  10 +-
 sysdeps/x86_64/strrchr.S                | 364 +-----------------------
 sysdeps/x86_64/wcsrchr.S                |  11 +-
 4 files changed, 366 insertions(+), 377 deletions(-)

Comments

H.J. Lu July 12, 2022, 10:28 p.m. UTC | #1
On Tue, Jul 12, 2022 at 12:29 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> This commit doesn't affect libc.so.6, its just housekeeping to prepare
> for adding explicit ISA level support.
>
> Tested build on x86_64 and x86_32 with/without multiarch.
> ---
>  sysdeps/x86_64/multiarch/strrchr-sse2.S | 358 ++++++++++++++++++++++-
>  sysdeps/x86_64/multiarch/wcsrchr-sse2.S |  10 +-
>  sysdeps/x86_64/strrchr.S                | 364 +-----------------------
>  sysdeps/x86_64/wcsrchr.S                |  11 +-
>  4 files changed, 366 insertions(+), 377 deletions(-)
>
> diff --git a/sysdeps/x86_64/multiarch/strrchr-sse2.S b/sysdeps/x86_64/multiarch/strrchr-sse2.S
> index 866396e947..6ee7a5e33a 100644
> --- a/sysdeps/x86_64/multiarch/strrchr-sse2.S
> +++ b/sysdeps/x86_64/multiarch/strrchr-sse2.S
> @@ -17,12 +17,358 @@
>     <https://www.gnu.org/licenses/>.  */
>
>  #if IS_IN (libc)
> -# define STRRCHR __strrchr_sse2
> +# ifndef STRRCHR
> +#  define STRRCHR __strrchr_sse2
> +# endif
> +#endif
> +
> +#include <sysdep.h>
> +
> +#ifdef USE_AS_WCSRCHR
> +# define PCMPEQ        pcmpeqd
> +# define CHAR_SIZE     4
> +# define PMINU pminud
> +#else
> +# define PCMPEQ        pcmpeqb
> +# define CHAR_SIZE     1
> +# define PMINU pminub
> +#endif
> +
> +#define PAGE_SIZE      4096
> +#define VEC_SIZE       16
> +
> +       .text
> +ENTRY(STRRCHR)
> +       movd    %esi, %xmm0
> +       movq    %rdi, %rax
> +       andl    $(PAGE_SIZE - 1), %eax
> +#ifndef USE_AS_WCSRCHR
> +       punpcklbw %xmm0, %xmm0
> +       punpcklwd %xmm0, %xmm0
> +#endif
> +       pshufd  $0, %xmm0, %xmm0
> +       cmpl    $(PAGE_SIZE - VEC_SIZE), %eax
> +       ja      L(cross_page)
> +
> +L(cross_page_continue):
> +       movups  (%rdi), %xmm1
> +       pxor    %xmm2, %xmm2
> +       PCMPEQ  %xmm1, %xmm2
> +       pmovmskb %xmm2, %ecx
> +       testl   %ecx, %ecx
> +       jz      L(aligned_more)
> +
> +       PCMPEQ  %xmm0, %xmm1
> +       pmovmskb %xmm1, %eax
> +       leal    -1(%rcx), %edx
> +       xorl    %edx, %ecx
> +       andl    %ecx, %eax
> +       jz      L(ret0)
> +       bsrl    %eax, %eax
> +       addq    %rdi, %rax
> +       /* We are off by 3 for wcsrchr if search CHAR is non-zero. If
> +          search CHAR is zero we are correct. Either way `andq
> +          -CHAR_SIZE, %rax` gets the correct result.  */
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +L(ret0):
> +       ret
> +
> +       /* Returns for first vec x1/x2 have hard coded backward search
> +          path for earlier matches.  */
> +       .p2align 4
> +L(first_vec_x0_test):
> +       PCMPEQ  %xmm0, %xmm1
> +       pmovmskb %xmm1, %eax
> +       testl   %eax, %eax
> +       jz      L(ret0)
> +       bsrl    %eax, %eax
> +       addq    %r8, %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       .p2align 4
> +L(first_vec_x1):
> +       PCMPEQ  %xmm0, %xmm2
> +       pmovmskb %xmm2, %eax
> +       leal    -1(%rcx), %edx
> +       xorl    %edx, %ecx
> +       andl    %ecx, %eax
> +       jz      L(first_vec_x0_test)
> +       bsrl    %eax, %eax
> +       leaq    (VEC_SIZE)(%rdi, %rax), %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       .p2align 4
> +L(first_vec_x1_test):
> +       PCMPEQ  %xmm0, %xmm2
> +       pmovmskb %xmm2, %eax
> +       testl   %eax, %eax
> +       jz      L(first_vec_x0_test)
> +       bsrl    %eax, %eax
> +       leaq    (VEC_SIZE)(%rdi, %rax), %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       .p2align 4
> +L(first_vec_x2):
> +       PCMPEQ  %xmm0, %xmm3
> +       pmovmskb %xmm3, %eax
> +       leal    -1(%rcx), %edx
> +       xorl    %edx, %ecx
> +       andl    %ecx, %eax
> +       jz      L(first_vec_x1_test)
> +       bsrl    %eax, %eax
> +       leaq    (VEC_SIZE * 2)(%rdi, %rax), %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       .p2align 4
> +L(aligned_more):
> +       /* Save original pointer if match was in VEC 0.  */
> +       movq    %rdi, %r8
> +       andq    $-VEC_SIZE, %rdi
> +
> +       movaps  VEC_SIZE(%rdi), %xmm2
> +       pxor    %xmm3, %xmm3
> +       PCMPEQ  %xmm2, %xmm3
> +       pmovmskb %xmm3, %ecx
> +       testl   %ecx, %ecx
> +       jnz     L(first_vec_x1)
> +
> +       movaps  (VEC_SIZE * 2)(%rdi), %xmm3
> +       pxor    %xmm4, %xmm4
> +       PCMPEQ  %xmm3, %xmm4
> +       pmovmskb %xmm4, %ecx
> +       testl   %ecx, %ecx
> +       jnz     L(first_vec_x2)
> +
> +       addq    $VEC_SIZE, %rdi
> +       /* Save pointer again before realigning.  */
> +       movq    %rdi, %rsi
> +       andq    $-(VEC_SIZE * 2), %rdi
> +       .p2align 4
> +L(first_loop):
> +       /* Do 2x VEC at a time.  */
> +       movaps  (VEC_SIZE * 2)(%rdi), %xmm4
> +       movaps  (VEC_SIZE * 3)(%rdi), %xmm5
> +       /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> +          detecting zero. Note if this is found to be a bottleneck it
> +          may be worth adding an SSE4.1 wcsrchr implementation.  */
> +#ifdef USE_AS_WCSRCHR
> +       movaps  %xmm5, %xmm6
> +       pxor    %xmm8, %xmm8
> +
> +       PCMPEQ  %xmm8, %xmm5
> +       PCMPEQ  %xmm4, %xmm8
> +       por     %xmm5, %xmm8
> +#else
> +       movaps  %xmm5, %xmm6
> +       PMINU   %xmm4, %xmm5
> +#endif
> +
> +       movaps  %xmm4, %xmm9
> +       PCMPEQ  %xmm0, %xmm4
> +       PCMPEQ  %xmm0, %xmm6
> +       movaps  %xmm6, %xmm7
> +       por     %xmm4, %xmm6
> +#ifndef USE_AS_WCSRCHR
> +       pxor    %xmm8, %xmm8
> +       PCMPEQ  %xmm5, %xmm8
> +#endif
> +       pmovmskb %xmm8, %ecx
> +       pmovmskb %xmm6, %eax
>
> -# undef weak_alias
> -# define weak_alias(strrchr, rindex)
> -# undef libc_hidden_builtin_def
> -# define libc_hidden_builtin_def(strrchr)
> +       addq    $(VEC_SIZE * 2), %rdi
> +       /* Use `addl` 1) so we can undo it with `subl` and 2) it can
> +          macro-fuse with `jz`.  */
> +       addl    %ecx, %eax
> +       jz      L(first_loop)
> +
> +       /* Check if there is zero match.  */
> +       testl   %ecx, %ecx
> +       jz      L(second_loop_match)
> +
> +       /* Check if there was a match in last iteration.  */
> +       subl    %ecx, %eax
> +       jnz     L(new_match)
> +
> +L(first_loop_old_match):
> +       PCMPEQ  %xmm0, %xmm2
> +       PCMPEQ  %xmm0, %xmm3
> +       pmovmskb %xmm2, %ecx
> +       pmovmskb %xmm3, %eax
> +       addl    %eax, %ecx
> +       jz      L(first_vec_x0_test)
> +       /* NB: We could move this shift to before the branch and save a
> +          bit of code size / performance on the fall through. The
> +          branch leads to the null case which generally seems hotter
> +          than char in first 3x VEC.  */
> +       sall    $16, %eax
> +       orl     %ecx, %eax
> +
> +       bsrl    %eax, %eax
> +       addq    %rsi, %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       .p2align 4
> +L(new_match):
> +       pxor    %xmm6, %xmm6
> +       PCMPEQ  %xmm9, %xmm6
> +       pmovmskb %xmm6, %eax
> +       sall    $16, %ecx
> +       orl     %eax, %ecx
> +
> +       /* We can't reuse either of the old comparisons as since we mask
> +          of zeros after first zero (instead of using the full
> +          comparison) we can't gurantee no interference between match
> +          after end of string and valid match.  */
> +       pmovmskb %xmm4, %eax
> +       pmovmskb %xmm7, %edx
> +       sall    $16, %edx
> +       orl     %edx, %eax
> +
> +       leal    -1(%ecx), %edx
> +       xorl    %edx, %ecx
> +       andl    %ecx, %eax
> +       jz      L(first_loop_old_match)
> +       bsrl    %eax, %eax
> +       addq    %rdi, %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       /* Save minimum state for getting most recent match. We can
> +          throw out all previous work.  */
> +       .p2align 4
> +L(second_loop_match):
> +       movq    %rdi, %rsi
> +       movaps  %xmm4, %xmm2
> +       movaps  %xmm7, %xmm3
> +
> +       .p2align 4
> +L(second_loop):
> +       movaps  (VEC_SIZE * 2)(%rdi), %xmm4
> +       movaps  (VEC_SIZE * 3)(%rdi), %xmm5
> +       /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> +          detecting zero. Note if this is found to be a bottleneck it
> +          may be worth adding an SSE4.1 wcsrchr implementation.  */
> +#ifdef USE_AS_WCSRCHR
> +       movaps  %xmm5, %xmm6
> +       pxor    %xmm8, %xmm8
> +
> +       PCMPEQ  %xmm8, %xmm5
> +       PCMPEQ  %xmm4, %xmm8
> +       por     %xmm5, %xmm8
> +#else
> +       movaps  %xmm5, %xmm6
> +       PMINU   %xmm4, %xmm5
> +#endif
> +
> +       movaps  %xmm4, %xmm9
> +       PCMPEQ  %xmm0, %xmm4
> +       PCMPEQ  %xmm0, %xmm6
> +       movaps  %xmm6, %xmm7
> +       por     %xmm4, %xmm6
> +#ifndef USE_AS_WCSRCHR
> +       pxor    %xmm8, %xmm8
> +       PCMPEQ  %xmm5, %xmm8
>  #endif
>
> -#include "../strrchr.S"
> +       pmovmskb %xmm8, %ecx
> +       pmovmskb %xmm6, %eax
> +
> +       addq    $(VEC_SIZE * 2), %rdi
> +       /* Either null term or new occurence of CHAR.  */
> +       addl    %ecx, %eax
> +       jz      L(second_loop)
> +
> +       /* No null term so much be new occurence of CHAR.  */
> +       testl   %ecx, %ecx
> +       jz      L(second_loop_match)
> +
> +
> +       subl    %ecx, %eax
> +       jnz     L(second_loop_new_match)
> +
> +L(second_loop_old_match):
> +       pmovmskb %xmm2, %ecx
> +       pmovmskb %xmm3, %eax
> +       sall    $16, %eax
> +       orl     %ecx, %eax
> +       bsrl    %eax, %eax
> +       addq    %rsi, %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       .p2align 4
> +L(second_loop_new_match):
> +       pxor    %xmm6, %xmm6
> +       PCMPEQ  %xmm9, %xmm6
> +       pmovmskb %xmm6, %eax
> +       sall    $16, %ecx
> +       orl     %eax, %ecx
> +
> +       /* We can't reuse either of the old comparisons as since we mask
> +          of zeros after first zero (instead of using the full
> +          comparison) we can't gurantee no interference between match
> +          after end of string and valid match.  */
> +       pmovmskb %xmm4, %eax
> +       pmovmskb %xmm7, %edx
> +       sall    $16, %edx
> +       orl     %edx, %eax
> +
> +       leal    -1(%ecx), %edx
> +       xorl    %edx, %ecx
> +       andl    %ecx, %eax
> +       jz      L(second_loop_old_match)
> +       bsrl    %eax, %eax
> +       addq    %rdi, %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +       ret
> +
> +       .p2align 4,, 4
> +L(cross_page):
> +       movq    %rdi, %rsi
> +       andq    $-VEC_SIZE, %rsi
> +       movaps  (%rsi), %xmm1
> +       pxor    %xmm2, %xmm2
> +       PCMPEQ  %xmm1, %xmm2
> +       pmovmskb %xmm2, %edx
> +       movl    %edi, %ecx
> +       andl    $(VEC_SIZE - 1), %ecx
> +       sarl    %cl, %edx
> +       jz      L(cross_page_continue)
> +       PCMPEQ  %xmm0, %xmm1
> +       pmovmskb %xmm1, %eax
> +       sarl    %cl, %eax
> +       leal    -1(%rdx), %ecx
> +       xorl    %edx, %ecx
> +       andl    %ecx, %eax
> +       jz      L(ret1)
> +       bsrl    %eax, %eax
> +       addq    %rdi, %rax
> +#ifdef USE_AS_WCSRCHR
> +       andq    $-CHAR_SIZE, %rax
> +#endif
> +L(ret1):
> +       ret
> +END(STRRCHR)
> diff --git a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
> index 69d2f3cdb1..d9259720f8 100644
> --- a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
> +++ b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
> @@ -17,6 +17,12 @@
>     <https://www.gnu.org/licenses/>.  */
>
>  #if IS_IN (libc)
> -# define STRRCHR       __wcsrchr_sse2
> +# ifndef STRRCHR
> +#  define STRRCHR      __wcsrchr_sse2
> +# endif
>  #endif
> -#include "../wcsrchr.S"
> +
> +#define USE_AS_WCSRCHR 1
> +#define NO_PMINU       1
> +
> +#include "strrchr-sse2.S"
> diff --git a/sysdeps/x86_64/strrchr.S b/sysdeps/x86_64/strrchr.S
> index 4d7ba4ceb2..f39da60454 100644
> --- a/sysdeps/x86_64/strrchr.S
> +++ b/sysdeps/x86_64/strrchr.S
> @@ -16,363 +16,7 @@
>     License along with the GNU C Library; if not, see
>     <https://www.gnu.org/licenses/>.  */
>
> -
> -#include <sysdep.h>
> -
> -#ifndef STRRCHR
> -# define STRRCHR       strrchr
> -#endif
> -
> -#ifdef USE_AS_WCSRCHR
> -# define PCMPEQ        pcmpeqd
> -# define CHAR_SIZE     4
> -# define PMINU pminud
> -#else
> -# define PCMPEQ        pcmpeqb
> -# define CHAR_SIZE     1
> -# define PMINU pminub
> -#endif
> -
> -#define PAGE_SIZE      4096
> -#define VEC_SIZE       16
> -
> -       .text
> -ENTRY(STRRCHR)
> -       movd    %esi, %xmm0
> -       movq    %rdi, %rax
> -       andl    $(PAGE_SIZE - 1), %eax
> -#ifndef USE_AS_WCSRCHR
> -       punpcklbw %xmm0, %xmm0
> -       punpcklwd %xmm0, %xmm0
> -#endif
> -       pshufd  $0, %xmm0, %xmm0
> -       cmpl    $(PAGE_SIZE - VEC_SIZE), %eax
> -       ja      L(cross_page)
> -
> -L(cross_page_continue):
> -       movups  (%rdi), %xmm1
> -       pxor    %xmm2, %xmm2
> -       PCMPEQ  %xmm1, %xmm2
> -       pmovmskb %xmm2, %ecx
> -       testl   %ecx, %ecx
> -       jz      L(aligned_more)
> -
> -       PCMPEQ  %xmm0, %xmm1
> -       pmovmskb %xmm1, %eax
> -       leal    -1(%rcx), %edx
> -       xorl    %edx, %ecx
> -       andl    %ecx, %eax
> -       jz      L(ret0)
> -       bsrl    %eax, %eax
> -       addq    %rdi, %rax
> -       /* We are off by 3 for wcsrchr if search CHAR is non-zero. If
> -          search CHAR is zero we are correct. Either way `andq
> -          -CHAR_SIZE, %rax` gets the correct result.  */
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -L(ret0):
> -       ret
> -
> -       /* Returns for first vec x1/x2 have hard coded backward search
> -          path for earlier matches.  */
> -       .p2align 4
> -L(first_vec_x0_test):
> -       PCMPEQ  %xmm0, %xmm1
> -       pmovmskb %xmm1, %eax
> -       testl   %eax, %eax
> -       jz      L(ret0)
> -       bsrl    %eax, %eax
> -       addq    %r8, %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       .p2align 4
> -L(first_vec_x1):
> -       PCMPEQ  %xmm0, %xmm2
> -       pmovmskb %xmm2, %eax
> -       leal    -1(%rcx), %edx
> -       xorl    %edx, %ecx
> -       andl    %ecx, %eax
> -       jz      L(first_vec_x0_test)
> -       bsrl    %eax, %eax
> -       leaq    (VEC_SIZE)(%rdi, %rax), %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       .p2align 4
> -L(first_vec_x1_test):
> -       PCMPEQ  %xmm0, %xmm2
> -       pmovmskb %xmm2, %eax
> -       testl   %eax, %eax
> -       jz      L(first_vec_x0_test)
> -       bsrl    %eax, %eax
> -       leaq    (VEC_SIZE)(%rdi, %rax), %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       .p2align 4
> -L(first_vec_x2):
> -       PCMPEQ  %xmm0, %xmm3
> -       pmovmskb %xmm3, %eax
> -       leal    -1(%rcx), %edx
> -       xorl    %edx, %ecx
> -       andl    %ecx, %eax
> -       jz      L(first_vec_x1_test)
> -       bsrl    %eax, %eax
> -       leaq    (VEC_SIZE * 2)(%rdi, %rax), %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       .p2align 4
> -L(aligned_more):
> -       /* Save original pointer if match was in VEC 0.  */
> -       movq    %rdi, %r8
> -       andq    $-VEC_SIZE, %rdi
> -
> -       movaps  VEC_SIZE(%rdi), %xmm2
> -       pxor    %xmm3, %xmm3
> -       PCMPEQ  %xmm2, %xmm3
> -       pmovmskb %xmm3, %ecx
> -       testl   %ecx, %ecx
> -       jnz     L(first_vec_x1)
> -
> -       movaps  (VEC_SIZE * 2)(%rdi), %xmm3
> -       pxor    %xmm4, %xmm4
> -       PCMPEQ  %xmm3, %xmm4
> -       pmovmskb %xmm4, %ecx
> -       testl   %ecx, %ecx
> -       jnz     L(first_vec_x2)
> -
> -       addq    $VEC_SIZE, %rdi
> -       /* Save pointer again before realigning.  */
> -       movq    %rdi, %rsi
> -       andq    $-(VEC_SIZE * 2), %rdi
> -       .p2align 4
> -L(first_loop):
> -       /* Do 2x VEC at a time.  */
> -       movaps  (VEC_SIZE * 2)(%rdi), %xmm4
> -       movaps  (VEC_SIZE * 3)(%rdi), %xmm5
> -       /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> -          detecting zero. Note if this is found to be a bottleneck it
> -          may be worth adding an SSE4.1 wcsrchr implementation.  */
> -#ifdef USE_AS_WCSRCHR
> -       movaps  %xmm5, %xmm6
> -       pxor    %xmm8, %xmm8
> -
> -       PCMPEQ  %xmm8, %xmm5
> -       PCMPEQ  %xmm4, %xmm8
> -       por     %xmm5, %xmm8
> -#else
> -       movaps  %xmm5, %xmm6
> -       PMINU   %xmm4, %xmm5
> -#endif
> -
> -       movaps  %xmm4, %xmm9
> -       PCMPEQ  %xmm0, %xmm4
> -       PCMPEQ  %xmm0, %xmm6
> -       movaps  %xmm6, %xmm7
> -       por     %xmm4, %xmm6
> -#ifndef USE_AS_WCSRCHR
> -       pxor    %xmm8, %xmm8
> -       PCMPEQ  %xmm5, %xmm8
> -#endif
> -       pmovmskb %xmm8, %ecx
> -       pmovmskb %xmm6, %eax
> -
> -       addq    $(VEC_SIZE * 2), %rdi
> -       /* Use `addl` 1) so we can undo it with `subl` and 2) it can
> -          macro-fuse with `jz`.  */
> -       addl    %ecx, %eax
> -       jz      L(first_loop)
> -
> -       /* Check if there is zero match.  */
> -       testl   %ecx, %ecx
> -       jz      L(second_loop_match)
> -
> -       /* Check if there was a match in last iteration.  */
> -       subl    %ecx, %eax
> -       jnz     L(new_match)
> -
> -L(first_loop_old_match):
> -       PCMPEQ  %xmm0, %xmm2
> -       PCMPEQ  %xmm0, %xmm3
> -       pmovmskb %xmm2, %ecx
> -       pmovmskb %xmm3, %eax
> -       addl    %eax, %ecx
> -       jz      L(first_vec_x0_test)
> -       /* NB: We could move this shift to before the branch and save a
> -          bit of code size / performance on the fall through. The
> -          branch leads to the null case which generally seems hotter
> -          than char in first 3x VEC.  */
> -       sall    $16, %eax
> -       orl     %ecx, %eax
> -
> -       bsrl    %eax, %eax
> -       addq    %rsi, %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       .p2align 4
> -L(new_match):
> -       pxor    %xmm6, %xmm6
> -       PCMPEQ  %xmm9, %xmm6
> -       pmovmskb %xmm6, %eax
> -       sall    $16, %ecx
> -       orl     %eax, %ecx
> -
> -       /* We can't reuse either of the old comparisons as since we mask
> -          of zeros after first zero (instead of using the full
> -          comparison) we can't gurantee no interference between match
> -          after end of string and valid match.  */
> -       pmovmskb %xmm4, %eax
> -       pmovmskb %xmm7, %edx
> -       sall    $16, %edx
> -       orl     %edx, %eax
> -
> -       leal    -1(%ecx), %edx
> -       xorl    %edx, %ecx
> -       andl    %ecx, %eax
> -       jz      L(first_loop_old_match)
> -       bsrl    %eax, %eax
> -       addq    %rdi, %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       /* Save minimum state for getting most recent match. We can
> -          throw out all previous work.  */
> -       .p2align 4
> -L(second_loop_match):
> -       movq    %rdi, %rsi
> -       movaps  %xmm4, %xmm2
> -       movaps  %xmm7, %xmm3
> -
> -       .p2align 4
> -L(second_loop):
> -       movaps  (VEC_SIZE * 2)(%rdi), %xmm4
> -       movaps  (VEC_SIZE * 3)(%rdi), %xmm5
> -       /* Since SSE2 no pminud so wcsrchr needs seperate logic for
> -          detecting zero. Note if this is found to be a bottleneck it
> -          may be worth adding an SSE4.1 wcsrchr implementation.  */
> -#ifdef USE_AS_WCSRCHR
> -       movaps  %xmm5, %xmm6
> -       pxor    %xmm8, %xmm8
> -
> -       PCMPEQ  %xmm8, %xmm5
> -       PCMPEQ  %xmm4, %xmm8
> -       por     %xmm5, %xmm8
> -#else
> -       movaps  %xmm5, %xmm6
> -       PMINU   %xmm4, %xmm5
> -#endif
> -
> -       movaps  %xmm4, %xmm9
> -       PCMPEQ  %xmm0, %xmm4
> -       PCMPEQ  %xmm0, %xmm6
> -       movaps  %xmm6, %xmm7
> -       por     %xmm4, %xmm6
> -#ifndef USE_AS_WCSRCHR
> -       pxor    %xmm8, %xmm8
> -       PCMPEQ  %xmm5, %xmm8
> -#endif
> -
> -       pmovmskb %xmm8, %ecx
> -       pmovmskb %xmm6, %eax
> -
> -       addq    $(VEC_SIZE * 2), %rdi
> -       /* Either null term or new occurence of CHAR.  */
> -       addl    %ecx, %eax
> -       jz      L(second_loop)
> -
> -       /* No null term so much be new occurence of CHAR.  */
> -       testl   %ecx, %ecx
> -       jz      L(second_loop_match)
> -
> -
> -       subl    %ecx, %eax
> -       jnz     L(second_loop_new_match)
> -
> -L(second_loop_old_match):
> -       pmovmskb %xmm2, %ecx
> -       pmovmskb %xmm3, %eax
> -       sall    $16, %eax
> -       orl     %ecx, %eax
> -       bsrl    %eax, %eax
> -       addq    %rsi, %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       .p2align 4
> -L(second_loop_new_match):
> -       pxor    %xmm6, %xmm6
> -       PCMPEQ  %xmm9, %xmm6
> -       pmovmskb %xmm6, %eax
> -       sall    $16, %ecx
> -       orl     %eax, %ecx
> -
> -       /* We can't reuse either of the old comparisons as since we mask
> -          of zeros after first zero (instead of using the full
> -          comparison) we can't gurantee no interference between match
> -          after end of string and valid match.  */
> -       pmovmskb %xmm4, %eax
> -       pmovmskb %xmm7, %edx
> -       sall    $16, %edx
> -       orl     %edx, %eax
> -
> -       leal    -1(%ecx), %edx
> -       xorl    %edx, %ecx
> -       andl    %ecx, %eax
> -       jz      L(second_loop_old_match)
> -       bsrl    %eax, %eax
> -       addq    %rdi, %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -       ret
> -
> -       .p2align 4,, 4
> -L(cross_page):
> -       movq    %rdi, %rsi
> -       andq    $-VEC_SIZE, %rsi
> -       movaps  (%rsi), %xmm1
> -       pxor    %xmm2, %xmm2
> -       PCMPEQ  %xmm1, %xmm2
> -       pmovmskb %xmm2, %edx
> -       movl    %edi, %ecx
> -       andl    $(VEC_SIZE - 1), %ecx
> -       sarl    %cl, %edx
> -       jz      L(cross_page_continue)
> -       PCMPEQ  %xmm0, %xmm1
> -       pmovmskb %xmm1, %eax
> -       sarl    %cl, %eax
> -       leal    -1(%rdx), %ecx
> -       xorl    %edx, %ecx
> -       andl    %ecx, %eax
> -       jz      L(ret1)
> -       bsrl    %eax, %eax
> -       addq    %rdi, %rax
> -#ifdef USE_AS_WCSRCHR
> -       andq    $-CHAR_SIZE, %rax
> -#endif
> -L(ret1):
> -       ret
> -END(STRRCHR)
> -
> -#ifndef USE_AS_WCSRCHR
> -       weak_alias (STRRCHR, rindex)
> -       libc_hidden_builtin_def (STRRCHR)
> -#endif
> +#define STRRCHR        strrchr
> +#include "multiarch/strrchr-sse2.S"
> +weak_alias (strrchr, rindex)
> +libc_hidden_builtin_def (strrchr)
> diff --git a/sysdeps/x86_64/wcsrchr.S b/sysdeps/x86_64/wcsrchr.S
> index 2b80efc5ef..1d4b1eb21c 100644
> --- a/sysdeps/x86_64/wcsrchr.S
> +++ b/sysdeps/x86_64/wcsrchr.S
> @@ -16,12 +16,5 @@
>     License along with the GNU C Library; if not, see
>     <https://www.gnu.org/licenses/>.  */
>
> -
> -#define USE_AS_WCSRCHR 1
> -#define NO_PMINU       1
> -
> -#ifndef STRRCHR
> -# define STRRCHR       wcsrchr
> -#endif
> -
> -#include "../strrchr.S"
> +#define STRRCHR        wcsrchr
> +#include "multiarch/wcsrchr-sse2.S"
> --
> 2.34.1
>

LGTM.

Thanks.
diff mbox series

Patch

diff --git a/sysdeps/x86_64/multiarch/strrchr-sse2.S b/sysdeps/x86_64/multiarch/strrchr-sse2.S
index 866396e947..6ee7a5e33a 100644
--- a/sysdeps/x86_64/multiarch/strrchr-sse2.S
+++ b/sysdeps/x86_64/multiarch/strrchr-sse2.S
@@ -17,12 +17,358 @@ 
    <https://www.gnu.org/licenses/>.  */
 
 #if IS_IN (libc)
-# define STRRCHR __strrchr_sse2
+# ifndef STRRCHR
+#  define STRRCHR __strrchr_sse2
+# endif
+#endif
+
+#include <sysdep.h>
+
+#ifdef USE_AS_WCSRCHR
+# define PCMPEQ	pcmpeqd
+# define CHAR_SIZE	4
+# define PMINU	pminud
+#else
+# define PCMPEQ	pcmpeqb
+# define CHAR_SIZE	1
+# define PMINU	pminub
+#endif
+
+#define PAGE_SIZE	4096
+#define VEC_SIZE	16
+
+	.text
+ENTRY(STRRCHR)
+	movd	%esi, %xmm0
+	movq	%rdi, %rax
+	andl	$(PAGE_SIZE - 1), %eax
+#ifndef USE_AS_WCSRCHR
+	punpcklbw %xmm0, %xmm0
+	punpcklwd %xmm0, %xmm0
+#endif
+	pshufd	$0, %xmm0, %xmm0
+	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
+	ja	L(cross_page)
+
+L(cross_page_continue):
+	movups	(%rdi), %xmm1
+	pxor	%xmm2, %xmm2
+	PCMPEQ	%xmm1, %xmm2
+	pmovmskb %xmm2, %ecx
+	testl	%ecx, %ecx
+	jz	L(aligned_more)
+
+	PCMPEQ	%xmm0, %xmm1
+	pmovmskb %xmm1, %eax
+	leal	-1(%rcx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(ret0)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+	/* We are off by 3 for wcsrchr if search CHAR is non-zero. If
+	   search CHAR is zero we are correct. Either way `andq
+	   -CHAR_SIZE, %rax` gets the correct result.  */
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+L(ret0):
+	ret
+
+	/* Returns for first vec x1/x2 have hard coded backward search
+	   path for earlier matches.  */
+	.p2align 4
+L(first_vec_x0_test):
+	PCMPEQ	%xmm0, %xmm1
+	pmovmskb %xmm1, %eax
+	testl	%eax, %eax
+	jz	L(ret0)
+	bsrl	%eax, %eax
+	addq	%r8, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(first_vec_x1):
+	PCMPEQ	%xmm0, %xmm2
+	pmovmskb %xmm2, %eax
+	leal	-1(%rcx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(first_vec_x0_test)
+	bsrl	%eax, %eax
+	leaq	(VEC_SIZE)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(first_vec_x1_test):
+	PCMPEQ	%xmm0, %xmm2
+	pmovmskb %xmm2, %eax
+	testl	%eax, %eax
+	jz	L(first_vec_x0_test)
+	bsrl	%eax, %eax
+	leaq	(VEC_SIZE)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(first_vec_x2):
+	PCMPEQ	%xmm0, %xmm3
+	pmovmskb %xmm3, %eax
+	leal	-1(%rcx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(first_vec_x1_test)
+	bsrl	%eax, %eax
+	leaq	(VEC_SIZE * 2)(%rdi, %rax), %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(aligned_more):
+	/* Save original pointer if match was in VEC 0.  */
+	movq	%rdi, %r8
+	andq	$-VEC_SIZE, %rdi
+
+	movaps	VEC_SIZE(%rdi), %xmm2
+	pxor	%xmm3, %xmm3
+	PCMPEQ	%xmm2, %xmm3
+	pmovmskb %xmm3, %ecx
+	testl	%ecx, %ecx
+	jnz	L(first_vec_x1)
+
+	movaps	(VEC_SIZE * 2)(%rdi), %xmm3
+	pxor	%xmm4, %xmm4
+	PCMPEQ	%xmm3, %xmm4
+	pmovmskb %xmm4, %ecx
+	testl	%ecx, %ecx
+	jnz	L(first_vec_x2)
+
+	addq	$VEC_SIZE, %rdi
+	/* Save pointer again before realigning.  */
+	movq	%rdi, %rsi
+	andq	$-(VEC_SIZE * 2), %rdi
+	.p2align 4
+L(first_loop):
+	/* Do 2x VEC at a time.  */
+	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
+	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
+	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
+	   detecting zero. Note if this is found to be a bottleneck it
+	   may be worth adding an SSE4.1 wcsrchr implementation.  */
+#ifdef USE_AS_WCSRCHR
+	movaps	%xmm5, %xmm6
+	pxor	%xmm8, %xmm8
+
+	PCMPEQ	%xmm8, %xmm5
+	PCMPEQ	%xmm4, %xmm8
+	por	%xmm5, %xmm8
+#else
+	movaps	%xmm5, %xmm6
+	PMINU	%xmm4, %xmm5
+#endif
+
+	movaps	%xmm4, %xmm9
+	PCMPEQ	%xmm0, %xmm4
+	PCMPEQ	%xmm0, %xmm6
+	movaps	%xmm6, %xmm7
+	por	%xmm4, %xmm6
+#ifndef USE_AS_WCSRCHR
+	pxor	%xmm8, %xmm8
+	PCMPEQ	%xmm5, %xmm8
+#endif
+	pmovmskb %xmm8, %ecx
+	pmovmskb %xmm6, %eax
 
-# undef weak_alias
-# define weak_alias(strrchr, rindex)
-# undef libc_hidden_builtin_def
-# define libc_hidden_builtin_def(strrchr)
+	addq	$(VEC_SIZE * 2), %rdi
+	/* Use `addl` 1) so we can undo it with `subl` and 2) it can
+	   macro-fuse with `jz`.  */
+	addl	%ecx, %eax
+	jz	L(first_loop)
+
+	/* Check if there is zero match.  */
+	testl	%ecx, %ecx
+	jz	L(second_loop_match)
+
+	/* Check if there was a match in last iteration.  */
+	subl	%ecx, %eax
+	jnz	L(new_match)
+
+L(first_loop_old_match):
+	PCMPEQ	%xmm0, %xmm2
+	PCMPEQ	%xmm0, %xmm3
+	pmovmskb %xmm2, %ecx
+	pmovmskb %xmm3, %eax
+	addl	%eax, %ecx
+	jz	L(first_vec_x0_test)
+	/* NB: We could move this shift to before the branch and save a
+	   bit of code size / performance on the fall through. The
+	   branch leads to the null case which generally seems hotter
+	   than char in first 3x VEC.  */
+	sall	$16, %eax
+	orl	%ecx, %eax
+
+	bsrl	%eax, %eax
+	addq	%rsi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(new_match):
+	pxor	%xmm6, %xmm6
+	PCMPEQ	%xmm9, %xmm6
+	pmovmskb %xmm6, %eax
+	sall	$16, %ecx
+	orl	%eax, %ecx
+
+	/* We can't reuse either of the old comparisons as since we mask
+	   of zeros after first zero (instead of using the full
+	   comparison) we can't gurantee no interference between match
+	   after end of string and valid match.  */
+	pmovmskb %xmm4, %eax
+	pmovmskb %xmm7, %edx
+	sall	$16, %edx
+	orl	%edx, %eax
+
+	leal	-1(%ecx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(first_loop_old_match)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	/* Save minimum state for getting most recent match. We can
+	   throw out all previous work.  */
+	.p2align 4
+L(second_loop_match):
+	movq	%rdi, %rsi
+	movaps	%xmm4, %xmm2
+	movaps	%xmm7, %xmm3
+
+	.p2align 4
+L(second_loop):
+	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
+	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
+	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
+	   detecting zero. Note if this is found to be a bottleneck it
+	   may be worth adding an SSE4.1 wcsrchr implementation.  */
+#ifdef USE_AS_WCSRCHR
+	movaps	%xmm5, %xmm6
+	pxor	%xmm8, %xmm8
+
+	PCMPEQ	%xmm8, %xmm5
+	PCMPEQ	%xmm4, %xmm8
+	por	%xmm5, %xmm8
+#else
+	movaps	%xmm5, %xmm6
+	PMINU	%xmm4, %xmm5
+#endif
+
+	movaps	%xmm4, %xmm9
+	PCMPEQ	%xmm0, %xmm4
+	PCMPEQ	%xmm0, %xmm6
+	movaps	%xmm6, %xmm7
+	por	%xmm4, %xmm6
+#ifndef USE_AS_WCSRCHR
+	pxor	%xmm8, %xmm8
+	PCMPEQ	%xmm5, %xmm8
 #endif
 
-#include "../strrchr.S"
+	pmovmskb %xmm8, %ecx
+	pmovmskb %xmm6, %eax
+
+	addq	$(VEC_SIZE * 2), %rdi
+	/* Either null term or new occurence of CHAR.  */
+	addl	%ecx, %eax
+	jz	L(second_loop)
+
+	/* No null term so much be new occurence of CHAR.  */
+	testl	%ecx, %ecx
+	jz	L(second_loop_match)
+
+
+	subl	%ecx, %eax
+	jnz	L(second_loop_new_match)
+
+L(second_loop_old_match):
+	pmovmskb %xmm2, %ecx
+	pmovmskb %xmm3, %eax
+	sall	$16, %eax
+	orl	%ecx, %eax
+	bsrl	%eax, %eax
+	addq	%rsi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4
+L(second_loop_new_match):
+	pxor	%xmm6, %xmm6
+	PCMPEQ	%xmm9, %xmm6
+	pmovmskb %xmm6, %eax
+	sall	$16, %ecx
+	orl	%eax, %ecx
+
+	/* We can't reuse either of the old comparisons as since we mask
+	   of zeros after first zero (instead of using the full
+	   comparison) we can't gurantee no interference between match
+	   after end of string and valid match.  */
+	pmovmskb %xmm4, %eax
+	pmovmskb %xmm7, %edx
+	sall	$16, %edx
+	orl	%edx, %eax
+
+	leal	-1(%ecx), %edx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(second_loop_old_match)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+	ret
+
+	.p2align 4,, 4
+L(cross_page):
+	movq	%rdi, %rsi
+	andq	$-VEC_SIZE, %rsi
+	movaps	(%rsi), %xmm1
+	pxor	%xmm2, %xmm2
+	PCMPEQ	%xmm1, %xmm2
+	pmovmskb %xmm2, %edx
+	movl	%edi, %ecx
+	andl	$(VEC_SIZE - 1), %ecx
+	sarl	%cl, %edx
+	jz	L(cross_page_continue)
+	PCMPEQ	%xmm0, %xmm1
+	pmovmskb %xmm1, %eax
+	sarl	%cl, %eax
+	leal	-1(%rdx), %ecx
+	xorl	%edx, %ecx
+	andl	%ecx, %eax
+	jz	L(ret1)
+	bsrl	%eax, %eax
+	addq	%rdi, %rax
+#ifdef USE_AS_WCSRCHR
+	andq	$-CHAR_SIZE, %rax
+#endif
+L(ret1):
+	ret
+END(STRRCHR)
diff --git a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
index 69d2f3cdb1..d9259720f8 100644
--- a/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
+++ b/sysdeps/x86_64/multiarch/wcsrchr-sse2.S
@@ -17,6 +17,12 @@ 
    <https://www.gnu.org/licenses/>.  */
 
 #if IS_IN (libc)
-# define STRRCHR	__wcsrchr_sse2
+# ifndef STRRCHR
+#  define STRRCHR	__wcsrchr_sse2
+# endif
 #endif
-#include "../wcsrchr.S"
+
+#define USE_AS_WCSRCHR	1
+#define NO_PMINU	1
+
+#include "strrchr-sse2.S"
diff --git a/sysdeps/x86_64/strrchr.S b/sysdeps/x86_64/strrchr.S
index 4d7ba4ceb2..f39da60454 100644
--- a/sysdeps/x86_64/strrchr.S
+++ b/sysdeps/x86_64/strrchr.S
@@ -16,363 +16,7 @@ 
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
-
-#include <sysdep.h>
-
-#ifndef STRRCHR
-# define STRRCHR	strrchr
-#endif
-
-#ifdef USE_AS_WCSRCHR
-# define PCMPEQ	pcmpeqd
-# define CHAR_SIZE	4
-# define PMINU	pminud
-#else
-# define PCMPEQ	pcmpeqb
-# define CHAR_SIZE	1
-# define PMINU	pminub
-#endif
-
-#define PAGE_SIZE	4096
-#define VEC_SIZE	16
-
-	.text
-ENTRY(STRRCHR)
-	movd	%esi, %xmm0
-	movq	%rdi, %rax
-	andl	$(PAGE_SIZE - 1), %eax
-#ifndef USE_AS_WCSRCHR
-	punpcklbw %xmm0, %xmm0
-	punpcklwd %xmm0, %xmm0
-#endif
-	pshufd	$0, %xmm0, %xmm0
-	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
-	ja	L(cross_page)
-
-L(cross_page_continue):
-	movups	(%rdi), %xmm1
-	pxor	%xmm2, %xmm2
-	PCMPEQ	%xmm1, %xmm2
-	pmovmskb %xmm2, %ecx
-	testl	%ecx, %ecx
-	jz	L(aligned_more)
-
-	PCMPEQ	%xmm0, %xmm1
-	pmovmskb %xmm1, %eax
-	leal	-1(%rcx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(ret0)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-	/* We are off by 3 for wcsrchr if search CHAR is non-zero. If
-	   search CHAR is zero we are correct. Either way `andq
-	   -CHAR_SIZE, %rax` gets the correct result.  */
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-L(ret0):
-	ret
-
-	/* Returns for first vec x1/x2 have hard coded backward search
-	   path for earlier matches.  */
-	.p2align 4
-L(first_vec_x0_test):
-	PCMPEQ	%xmm0, %xmm1
-	pmovmskb %xmm1, %eax
-	testl	%eax, %eax
-	jz	L(ret0)
-	bsrl	%eax, %eax
-	addq	%r8, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(first_vec_x1):
-	PCMPEQ	%xmm0, %xmm2
-	pmovmskb %xmm2, %eax
-	leal	-1(%rcx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(first_vec_x0_test)
-	bsrl	%eax, %eax
-	leaq	(VEC_SIZE)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(first_vec_x1_test):
-	PCMPEQ	%xmm0, %xmm2
-	pmovmskb %xmm2, %eax
-	testl	%eax, %eax
-	jz	L(first_vec_x0_test)
-	bsrl	%eax, %eax
-	leaq	(VEC_SIZE)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(first_vec_x2):
-	PCMPEQ	%xmm0, %xmm3
-	pmovmskb %xmm3, %eax
-	leal	-1(%rcx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(first_vec_x1_test)
-	bsrl	%eax, %eax
-	leaq	(VEC_SIZE * 2)(%rdi, %rax), %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(aligned_more):
-	/* Save original pointer if match was in VEC 0.  */
-	movq	%rdi, %r8
-	andq	$-VEC_SIZE, %rdi
-
-	movaps	VEC_SIZE(%rdi), %xmm2
-	pxor	%xmm3, %xmm3
-	PCMPEQ	%xmm2, %xmm3
-	pmovmskb %xmm3, %ecx
-	testl	%ecx, %ecx
-	jnz	L(first_vec_x1)
-
-	movaps	(VEC_SIZE * 2)(%rdi), %xmm3
-	pxor	%xmm4, %xmm4
-	PCMPEQ	%xmm3, %xmm4
-	pmovmskb %xmm4, %ecx
-	testl	%ecx, %ecx
-	jnz	L(first_vec_x2)
-
-	addq	$VEC_SIZE, %rdi
-	/* Save pointer again before realigning.  */
-	movq	%rdi, %rsi
-	andq	$-(VEC_SIZE * 2), %rdi
-	.p2align 4
-L(first_loop):
-	/* Do 2x VEC at a time.  */
-	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
-	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
-	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
-	   detecting zero. Note if this is found to be a bottleneck it
-	   may be worth adding an SSE4.1 wcsrchr implementation.  */
-#ifdef USE_AS_WCSRCHR
-	movaps	%xmm5, %xmm6
-	pxor	%xmm8, %xmm8
-
-	PCMPEQ	%xmm8, %xmm5
-	PCMPEQ	%xmm4, %xmm8
-	por	%xmm5, %xmm8
-#else
-	movaps	%xmm5, %xmm6
-	PMINU	%xmm4, %xmm5
-#endif
-
-	movaps	%xmm4, %xmm9
-	PCMPEQ	%xmm0, %xmm4
-	PCMPEQ	%xmm0, %xmm6
-	movaps	%xmm6, %xmm7
-	por	%xmm4, %xmm6
-#ifndef USE_AS_WCSRCHR
-	pxor	%xmm8, %xmm8
-	PCMPEQ	%xmm5, %xmm8
-#endif
-	pmovmskb %xmm8, %ecx
-	pmovmskb %xmm6, %eax
-
-	addq	$(VEC_SIZE * 2), %rdi
-	/* Use `addl` 1) so we can undo it with `subl` and 2) it can
-	   macro-fuse with `jz`.  */
-	addl	%ecx, %eax
-	jz	L(first_loop)
-
-	/* Check if there is zero match.  */
-	testl	%ecx, %ecx
-	jz	L(second_loop_match)
-
-	/* Check if there was a match in last iteration.  */
-	subl	%ecx, %eax
-	jnz	L(new_match)
-
-L(first_loop_old_match):
-	PCMPEQ	%xmm0, %xmm2
-	PCMPEQ	%xmm0, %xmm3
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm3, %eax
-	addl	%eax, %ecx
-	jz	L(first_vec_x0_test)
-	/* NB: We could move this shift to before the branch and save a
-	   bit of code size / performance on the fall through. The
-	   branch leads to the null case which generally seems hotter
-	   than char in first 3x VEC.  */
-	sall	$16, %eax
-	orl	%ecx, %eax
-
-	bsrl	%eax, %eax
-	addq	%rsi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(new_match):
-	pxor	%xmm6, %xmm6
-	PCMPEQ	%xmm9, %xmm6
-	pmovmskb %xmm6, %eax
-	sall	$16, %ecx
-	orl	%eax, %ecx
-
-	/* We can't reuse either of the old comparisons as since we mask
-	   of zeros after first zero (instead of using the full
-	   comparison) we can't gurantee no interference between match
-	   after end of string and valid match.  */
-	pmovmskb %xmm4, %eax
-	pmovmskb %xmm7, %edx
-	sall	$16, %edx
-	orl	%edx, %eax
-
-	leal	-1(%ecx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(first_loop_old_match)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	/* Save minimum state for getting most recent match. We can
-	   throw out all previous work.  */
-	.p2align 4
-L(second_loop_match):
-	movq	%rdi, %rsi
-	movaps	%xmm4, %xmm2
-	movaps	%xmm7, %xmm3
-
-	.p2align 4
-L(second_loop):
-	movaps	(VEC_SIZE * 2)(%rdi), %xmm4
-	movaps	(VEC_SIZE * 3)(%rdi), %xmm5
-	/* Since SSE2 no pminud so wcsrchr needs seperate logic for
-	   detecting zero. Note if this is found to be a bottleneck it
-	   may be worth adding an SSE4.1 wcsrchr implementation.  */
-#ifdef USE_AS_WCSRCHR
-	movaps	%xmm5, %xmm6
-	pxor	%xmm8, %xmm8
-
-	PCMPEQ	%xmm8, %xmm5
-	PCMPEQ	%xmm4, %xmm8
-	por	%xmm5, %xmm8
-#else
-	movaps	%xmm5, %xmm6
-	PMINU	%xmm4, %xmm5
-#endif
-
-	movaps	%xmm4, %xmm9
-	PCMPEQ	%xmm0, %xmm4
-	PCMPEQ	%xmm0, %xmm6
-	movaps	%xmm6, %xmm7
-	por	%xmm4, %xmm6
-#ifndef USE_AS_WCSRCHR
-	pxor	%xmm8, %xmm8
-	PCMPEQ	%xmm5, %xmm8
-#endif
-
-	pmovmskb %xmm8, %ecx
-	pmovmskb %xmm6, %eax
-
-	addq	$(VEC_SIZE * 2), %rdi
-	/* Either null term or new occurence of CHAR.  */
-	addl	%ecx, %eax
-	jz	L(second_loop)
-
-	/* No null term so much be new occurence of CHAR.  */
-	testl	%ecx, %ecx
-	jz	L(second_loop_match)
-
-
-	subl	%ecx, %eax
-	jnz	L(second_loop_new_match)
-
-L(second_loop_old_match):
-	pmovmskb %xmm2, %ecx
-	pmovmskb %xmm3, %eax
-	sall	$16, %eax
-	orl	%ecx, %eax
-	bsrl	%eax, %eax
-	addq	%rsi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4
-L(second_loop_new_match):
-	pxor	%xmm6, %xmm6
-	PCMPEQ	%xmm9, %xmm6
-	pmovmskb %xmm6, %eax
-	sall	$16, %ecx
-	orl	%eax, %ecx
-
-	/* We can't reuse either of the old comparisons as since we mask
-	   of zeros after first zero (instead of using the full
-	   comparison) we can't gurantee no interference between match
-	   after end of string and valid match.  */
-	pmovmskb %xmm4, %eax
-	pmovmskb %xmm7, %edx
-	sall	$16, %edx
-	orl	%edx, %eax
-
-	leal	-1(%ecx), %edx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(second_loop_old_match)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-	ret
-
-	.p2align 4,, 4
-L(cross_page):
-	movq	%rdi, %rsi
-	andq	$-VEC_SIZE, %rsi
-	movaps	(%rsi), %xmm1
-	pxor	%xmm2, %xmm2
-	PCMPEQ	%xmm1, %xmm2
-	pmovmskb %xmm2, %edx
-	movl	%edi, %ecx
-	andl	$(VEC_SIZE - 1), %ecx
-	sarl	%cl, %edx
-	jz	L(cross_page_continue)
-	PCMPEQ	%xmm0, %xmm1
-	pmovmskb %xmm1, %eax
-	sarl	%cl, %eax
-	leal	-1(%rdx), %ecx
-	xorl	%edx, %ecx
-	andl	%ecx, %eax
-	jz	L(ret1)
-	bsrl	%eax, %eax
-	addq	%rdi, %rax
-#ifdef USE_AS_WCSRCHR
-	andq	$-CHAR_SIZE, %rax
-#endif
-L(ret1):
-	ret
-END(STRRCHR)
-
-#ifndef USE_AS_WCSRCHR
-	weak_alias (STRRCHR, rindex)
-	libc_hidden_builtin_def (STRRCHR)
-#endif
+#define STRRCHR	strrchr
+#include "multiarch/strrchr-sse2.S"
+weak_alias (strrchr, rindex)
+libc_hidden_builtin_def (strrchr)
diff --git a/sysdeps/x86_64/wcsrchr.S b/sysdeps/x86_64/wcsrchr.S
index 2b80efc5ef..1d4b1eb21c 100644
--- a/sysdeps/x86_64/wcsrchr.S
+++ b/sysdeps/x86_64/wcsrchr.S
@@ -16,12 +16,5 @@ 
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
-
-#define USE_AS_WCSRCHR	1
-#define NO_PMINU	1
-
-#ifndef STRRCHR
-# define STRRCHR	wcsrchr
-#endif
-
-#include "../strrchr.S"
+#define STRRCHR	wcsrchr
+#include "multiarch/wcsrchr-sse2.S"