| Message ID | 20220713233301.1868315-3-goldstein.w.n@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | [v2,1/3] x86: Add support to build strcmp/strlen/strchr with explicit ISA level | expand |
On Wed, Jul 13, 2022 at 4:33 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > 1. Add default ISA level selection in non-multiarch/rtld > implementations. > > 2. Add ISA level build guards to different implementations. > - I.e strcpy-avx2.S which is ISA level 3 will only build if > compiled ISA level <= 3. Otherwise there is no reason to > include it as we will always use one of the ISA level 4 > implementations (strcpy-evex.S). > > 3. Refactor the ifunc selector and ifunc implementation list to use > the ISA level aware wrapper macros that allow functions below the > compiled ISA level (with a guranteed replacement) to be skipped. > > Tested with and without multiarch on x86_64 for ISA levels: > {generic, x86-64-v2, x86-64-v3, x86-64-v4} > > And m32 with and without multiarch. > --- > sysdeps/x86_64/multiarch/ifunc-impl-list.c | 183 +++++++++++------- > sysdeps/x86_64/multiarch/ifunc-strcpy.h | 27 +-- > sysdeps/x86_64/multiarch/ifunc-strncpy.h | 22 ++- > sysdeps/x86_64/multiarch/stpcpy-avx2.S | 6 +- > sysdeps/x86_64/multiarch/stpcpy-evex.S | 6 +- > .../x86_64/multiarch/stpcpy-sse2-unaligned.S | 6 +- > sysdeps/x86_64/multiarch/stpcpy-sse2.S | 25 +-- > sysdeps/x86_64/multiarch/stpncpy-avx2.S | 6 +- > sysdeps/x86_64/multiarch/stpncpy-evex.S | 6 +- > .../x86_64/multiarch/stpncpy-sse2-unaligned.S | 6 +- > sysdeps/x86_64/multiarch/strcat-avx2.S | 5 +- > sysdeps/x86_64/multiarch/strcat-evex.S | 5 +- > .../x86_64/multiarch/strcat-sse2-unaligned.S | 7 +- > sysdeps/x86_64/multiarch/strcat-sse2.S | 18 +- > sysdeps/x86_64/multiarch/strcpy-avx2.S | 5 +- > sysdeps/x86_64/multiarch/strcpy-evex.S | 5 +- > .../x86_64/multiarch/strcpy-sse2-unaligned.S | 7 +- > sysdeps/x86_64/multiarch/strcpy-sse2.S | 23 ++- > sysdeps/x86_64/multiarch/strncat-avx2.S | 6 +- > sysdeps/x86_64/multiarch/strncat-evex.S | 6 +- > .../x86_64/multiarch/strncat-sse2-unaligned.S | 6 +- > sysdeps/x86_64/multiarch/strncpy-avx2.S | 6 +- > sysdeps/x86_64/multiarch/strncpy-evex.S | 6 +- > .../x86_64/multiarch/strncpy-sse2-unaligned.S | 6 +- > sysdeps/x86_64/stpcpy.S | 26 ++- > sysdeps/x86_64/stpncpy.S | 28 +++ > sysdeps/x86_64/strcat.S | 15 +- > sysdeps/x86_64/strcpy.S | 12 +- > sysdeps/x86_64/strncat.S | 28 +++ > sysdeps/x86_64/strncpy.S | 27 +++ > 30 files changed, 382 insertions(+), 158 deletions(-) > create mode 100644 sysdeps/x86_64/stpncpy.S > create mode 100644 sysdeps/x86_64/strncat.S > create mode 100644 sysdeps/x86_64/strncpy.S > > diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c > index 9318e98cc8..a71444eccb 100644 > --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c > +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c > @@ -403,33 +403,46 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, > > /* Support sysdeps/x86_64/multiarch/stpncpy.c. */ > IFUNC_IMPL (i, name, stpncpy, > - IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2), > - __stpncpy_avx2) > - IFUNC_IMPL_ADD (array, i, stpncpy, > - (CPU_FEATURE_USABLE (AVX2) > - && CPU_FEATURE_USABLE (RTM)), > - __stpncpy_avx2_rtm) > - IFUNC_IMPL_ADD (array, i, stpncpy, > - (CPU_FEATURE_USABLE (AVX512VL) > - && CPU_FEATURE_USABLE (AVX512BW)), > - __stpncpy_evex) > - IFUNC_IMPL_ADD (array, i, stpncpy, 1, > - __stpncpy_sse2_unaligned)) > + X86_IFUNC_IMPL_ADD_V4 (array, i, stpncpy, > + (CPU_FEATURE_USABLE (AVX512VL) > + && CPU_FEATURE_USABLE (AVX512BW)), > + __stpncpy_evex) > + X86_IFUNC_IMPL_ADD_V3 (array, i, stpncpy, > + CPU_FEATURE_USABLE (AVX2), > + __stpncpy_avx2) > + X86_IFUNC_IMPL_ADD_V3 (array, i, stpncpy, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (RTM)), > + __stpncpy_avx2_rtm) > + /* ISA V2 wrapper for sse2_unaligned implementation because > + the sse2_unaligned implementation is also used at ISA > + level 2. */ > + X86_IFUNC_IMPL_ADD_V2 (array, i, stpncpy, > + 1, > + __stpncpy_sse2_unaligned)) > > /* Support sysdeps/x86_64/multiarch/stpcpy.c. */ > IFUNC_IMPL (i, name, stpcpy, > - IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2), > - __stpcpy_avx2) > - IFUNC_IMPL_ADD (array, i, stpcpy, > - (CPU_FEATURE_USABLE (AVX2) > - && CPU_FEATURE_USABLE (RTM)), > - __stpcpy_avx2_rtm) > - IFUNC_IMPL_ADD (array, i, stpcpy, > - (CPU_FEATURE_USABLE (AVX512VL) > - && CPU_FEATURE_USABLE (AVX512BW)), > - __stpcpy_evex) > - IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned) > - IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2)) > + X86_IFUNC_IMPL_ADD_V4 (array, i, stpcpy, > + (CPU_FEATURE_USABLE (AVX512VL) > + && CPU_FEATURE_USABLE (AVX512BW)), > + __stpcpy_evex) > + X86_IFUNC_IMPL_ADD_V3 (array, i, stpcpy, > + CPU_FEATURE_USABLE (AVX2), > + __stpcpy_avx2) > + X86_IFUNC_IMPL_ADD_V3 (array, i, stpcpy, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (RTM)), > + __stpcpy_avx2_rtm) > + /* ISA V2 wrapper for sse2_unaligned implementation because > + the sse2_unaligned implementation is also used at ISA > + level 2. */ > + X86_IFUNC_IMPL_ADD_V2 (array, i, stpcpy, > + 1, > + __stpcpy_sse2_unaligned) > + X86_IFUNC_IMPL_ADD_V1 (array, i, stpcpy, > + 1, > + __stpcpy_sse2)) > > /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */ > IFUNC_IMPL (i, name, strcasecmp, > @@ -477,18 +490,26 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, > > /* Support sysdeps/x86_64/multiarch/strcat.c. */ > IFUNC_IMPL (i, name, strcat, > - IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2), > - __strcat_avx2) > - IFUNC_IMPL_ADD (array, i, strcat, > - (CPU_FEATURE_USABLE (AVX2) > - && CPU_FEATURE_USABLE (RTM)), > - __strcat_avx2_rtm) > - IFUNC_IMPL_ADD (array, i, strcat, > - (CPU_FEATURE_USABLE (AVX512VL) > - && CPU_FEATURE_USABLE (AVX512BW)), > - __strcat_evex) > - IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned) > - IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2)) > + X86_IFUNC_IMPL_ADD_V4 (array, i, strcat, > + (CPU_FEATURE_USABLE (AVX512VL) > + && CPU_FEATURE_USABLE (AVX512BW)), > + __strcat_evex) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strcat, > + CPU_FEATURE_USABLE (AVX2), > + __strcat_avx2) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strcat, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (RTM)), > + __strcat_avx2_rtm) > + /* ISA V2 wrapper for sse2_unaligned implementation because > + the sse2_unaligned implementation is also used at ISA > + level 2. */ > + X86_IFUNC_IMPL_ADD_V2 (array, i, strcat, > + 1, > + __strcat_sse2_unaligned) > + X86_IFUNC_IMPL_ADD_V1 (array, i, strcat, > + 1, > + __strcat_sse2)) > > /* Support sysdeps/x86_64/multiarch/strchr.c. */ > IFUNC_IMPL (i, name, strchr, > @@ -584,18 +605,26 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, > > /* Support sysdeps/x86_64/multiarch/strcpy.c. */ > IFUNC_IMPL (i, name, strcpy, > - IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2), > - __strcpy_avx2) > - IFUNC_IMPL_ADD (array, i, strcpy, > - (CPU_FEATURE_USABLE (AVX2) > - && CPU_FEATURE_USABLE (RTM)), > - __strcpy_avx2_rtm) > - IFUNC_IMPL_ADD (array, i, strcpy, > - (CPU_FEATURE_USABLE (AVX512VL) > - && CPU_FEATURE_USABLE (AVX512BW)), > - __strcpy_evex) > - IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned) > - IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2)) > + X86_IFUNC_IMPL_ADD_V4 (array, i, strcpy, > + (CPU_FEATURE_USABLE (AVX512VL) > + && CPU_FEATURE_USABLE (AVX512BW)), > + __strcpy_evex) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strcpy, > + CPU_FEATURE_USABLE (AVX2), > + __strcpy_avx2) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strcpy, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (RTM)), > + __strcpy_avx2_rtm) > + /* ISA V2 wrapper for sse2_unaligned implementation because > + the sse2_unaligned implementation is also used at ISA > + level 2. */ > + X86_IFUNC_IMPL_ADD_V2 (array, i, strcpy, > + 1, > + __strcpy_sse2_unaligned) > + X86_IFUNC_IMPL_ADD_V1 (array, i, strcpy, > + 1, > + __strcpy_sse2)) > > /* Support sysdeps/x86_64/multiarch/strcspn.c. */ > IFUNC_IMPL (i, name, strcspn, > @@ -651,33 +680,43 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, > > /* Support sysdeps/x86_64/multiarch/strncat.c. */ > IFUNC_IMPL (i, name, strncat, > - IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2), > - __strncat_avx2) > - IFUNC_IMPL_ADD (array, i, strncat, > - (CPU_FEATURE_USABLE (AVX2) > - && CPU_FEATURE_USABLE (RTM)), > - __strncat_avx2_rtm) > - IFUNC_IMPL_ADD (array, i, strncat, > - (CPU_FEATURE_USABLE (AVX512VL) > - && CPU_FEATURE_USABLE (AVX512BW)), > - __strncat_evex) > - IFUNC_IMPL_ADD (array, i, strncat, 1, > - __strncat_sse2_unaligned)) > + X86_IFUNC_IMPL_ADD_V4 (array, i, strncat, > + (CPU_FEATURE_USABLE (AVX512VL) > + && CPU_FEATURE_USABLE (AVX512BW)), > + __strncat_evex) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strncat, > + CPU_FEATURE_USABLE (AVX2), > + __strncat_avx2) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strncat, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (RTM)), > + __strncat_avx2_rtm) > + /* ISA V2 wrapper for sse2_unaligned implementation because > + the sse2_unaligned implementation is also used at ISA > + level 2. */ > + X86_IFUNC_IMPL_ADD_V2 (array, i, strncat, > + 1, > + __strncat_sse2_unaligned)) > > /* Support sysdeps/x86_64/multiarch/strncpy.c. */ > IFUNC_IMPL (i, name, strncpy, > - IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2), > - __strncpy_avx2) > - IFUNC_IMPL_ADD (array, i, strncpy, > - (CPU_FEATURE_USABLE (AVX2) > - && CPU_FEATURE_USABLE (RTM)), > - __strncpy_avx2_rtm) > - IFUNC_IMPL_ADD (array, i, strncpy, > - (CPU_FEATURE_USABLE (AVX512VL) > - && CPU_FEATURE_USABLE (AVX512BW)), > - __strncpy_evex) > - IFUNC_IMPL_ADD (array, i, strncpy, 1, > - __strncpy_sse2_unaligned)) > + X86_IFUNC_IMPL_ADD_V4 (array, i, strncpy, > + (CPU_FEATURE_USABLE (AVX512VL) > + && CPU_FEATURE_USABLE (AVX512BW)), > + __strncpy_evex) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strncpy, > + CPU_FEATURE_USABLE (AVX2), > + __strncpy_avx2) > + X86_IFUNC_IMPL_ADD_V3 (array, i, strncpy, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (RTM)), > + __strncpy_avx2_rtm) > + /* ISA V2 wrapper for sse2_unaligned implementation because > + the sse2_unaligned implementation is also used at ISA > + level 2. */ > + X86_IFUNC_IMPL_ADD_V2 (array, i, strncpy, > + 1, > + __strncpy_sse2_unaligned)) > > /* Support sysdeps/x86_64/multiarch/strpbrk.c. */ > IFUNC_IMPL (i, name, strpbrk, > diff --git a/sysdeps/x86_64/multiarch/ifunc-strcpy.h b/sysdeps/x86_64/multiarch/ifunc-strcpy.h > index a15afa44e9..e083f71df3 100644 > --- a/sysdeps/x86_64/multiarch/ifunc-strcpy.h > +++ b/sysdeps/x86_64/multiarch/ifunc-strcpy.h > @@ -20,33 +20,38 @@ > > #include <init-arch.h> > > -extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; > -extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) > - attribute_hidden; > +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; > + > extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; > extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; > -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; > + > +extern __typeof (REDIRECT_NAME) > + OPTIMIZE (sse2_unaligned) attribute_hidden; > + > +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; > > static inline void * > IFUNC_SELECTOR (void) > { > - const struct cpu_features* cpu_features = __get_cpu_features (); > + const struct cpu_features *cpu_features = __get_cpu_features (); > > - if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) > - && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) > + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2) > + && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, > + AVX_Fast_Unaligned_Load, )) > { > - if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) > - && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) > + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) > + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) > return OPTIMIZE (evex); > > if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) > return OPTIMIZE (avx2_rtm); > > - if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) > + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, > + Prefer_No_VZEROUPPER, !)) > return OPTIMIZE (avx2); > } > > - if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load)) > + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load, )) > return OPTIMIZE (sse2_unaligned); > > return OPTIMIZE (sse2); > diff --git a/sysdeps/x86_64/multiarch/ifunc-strncpy.h b/sysdeps/x86_64/multiarch/ifunc-strncpy.h > index 323225af4d..5c1c46b885 100644 > --- a/sysdeps/x86_64/multiarch/ifunc-strncpy.h > +++ b/sysdeps/x86_64/multiarch/ifunc-strncpy.h > @@ -19,28 +19,32 @@ > > #include <init-arch.h> > > -extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) > - attribute_hidden; > +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; > + > extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; > extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; > -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; > + > +extern __typeof (REDIRECT_NAME) > + OPTIMIZE (sse2_unaligned) attribute_hidden; > > static inline void * > IFUNC_SELECTOR (void) > { > - const struct cpu_features* cpu_features = __get_cpu_features (); > + const struct cpu_features *cpu_features = __get_cpu_features (); > > - if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) > - && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) > + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2) > + && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, > + AVX_Fast_Unaligned_Load, )) > { > - if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) > - && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) > + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) > + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) > return OPTIMIZE (evex); > > if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) > return OPTIMIZE (avx2_rtm); > > - if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) > + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, > + Prefer_No_VZEROUPPER, !)) > return OPTIMIZE (avx2); > } > > diff --git a/sysdeps/x86_64/multiarch/stpcpy-avx2.S b/sysdeps/x86_64/multiarch/stpcpy-avx2.S > index f0bd3029fe..277f9e73d2 100644 > --- a/sysdeps/x86_64/multiarch/stpcpy-avx2.S > +++ b/sysdeps/x86_64/multiarch/stpcpy-avx2.S > @@ -1,3 +1,7 @@ > +#ifndef STPCPY > +# define STPCPY __stpcpy_avx2 > +#endif > + > #define USE_AS_STPCPY > -#define STRCPY __stpcpy_avx2 > +#define STRCPY STPCPY > #include "strcpy-avx2.S" > diff --git a/sysdeps/x86_64/multiarch/stpcpy-evex.S b/sysdeps/x86_64/multiarch/stpcpy-evex.S > index 7c6f26cd98..4f1c015424 100644 > --- a/sysdeps/x86_64/multiarch/stpcpy-evex.S > +++ b/sysdeps/x86_64/multiarch/stpcpy-evex.S > @@ -1,3 +1,7 @@ > +#ifndef STPCPY > +# define STPCPY __stpcpy_evex > +#endif > + > #define USE_AS_STPCPY > -#define STRCPY __stpcpy_evex > +#define STRCPY STPCPY > #include "strcpy-evex.S" > diff --git a/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S > index 34231f8b46..4c77e5b51c 100644 > --- a/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S > +++ b/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S > @@ -1,3 +1,7 @@ > +#ifndef STPCPY > +# define STPCPY __stpcpy_sse2_unaligned > +#endif > + > #define USE_AS_STPCPY > -#define STRCPY __stpcpy_sse2_unaligned > +#define STRCPY STPCPY > #include "strcpy-sse2-unaligned.S" > diff --git a/sysdeps/x86_64/multiarch/stpcpy-sse2.S b/sysdeps/x86_64/multiarch/stpcpy-sse2.S > index ea9f973af3..fcd67cada2 100644 > --- a/sysdeps/x86_64/multiarch/stpcpy-sse2.S > +++ b/sysdeps/x86_64/multiarch/stpcpy-sse2.S > @@ -1,26 +1,7 @@ > -/* stpcpy optimized with SSE2. > - Copyright (C) 2017-2022 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <https://www.gnu.org/licenses/>. */ > - > -#if IS_IN (libc) > -# ifndef STRCPY > -# define STRCPY __stpcpy_sse2 > -# endif > +#ifndef STPCPY > +# define STPCPY __stpcpy_sse2 > #endif > > #define USE_AS_STPCPY > +#define STRCPY STPCPY > #include "strcpy-sse2.S" > diff --git a/sysdeps/x86_64/multiarch/stpncpy-avx2.S b/sysdeps/x86_64/multiarch/stpncpy-avx2.S > index 032b0407d0..b2f8c19143 100644 > --- a/sysdeps/x86_64/multiarch/stpncpy-avx2.S > +++ b/sysdeps/x86_64/multiarch/stpncpy-avx2.S > @@ -1,4 +1,8 @@ > +#ifndef STPNCPY > +# define STPNCPY __stpncpy_avx2 > +#endif > + > #define USE_AS_STPCPY > #define USE_AS_STRNCPY > -#define STRCPY __stpncpy_avx2 > +#define STRCPY STPNCPY > #include "strcpy-avx2.S" > diff --git a/sysdeps/x86_64/multiarch/stpncpy-evex.S b/sysdeps/x86_64/multiarch/stpncpy-evex.S > index 1570014d1c..99ea76a372 100644 > --- a/sysdeps/x86_64/multiarch/stpncpy-evex.S > +++ b/sysdeps/x86_64/multiarch/stpncpy-evex.S > @@ -1,4 +1,8 @@ > +#ifndef STPNCPY > +# define STPNCPY __stpncpy_evex > +#endif > + > #define USE_AS_STPCPY > #define USE_AS_STRNCPY > -#define STRCPY __stpncpy_evex > +#define STRCPY STPNCPY > #include "strcpy-evex.S" > diff --git a/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S > index 658520f78f..c83706016f 100644 > --- a/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S > +++ b/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S > @@ -1,4 +1,8 @@ > +#ifndef STPNCPY > +# define STPNCPY __stpncpy_sse2_unaligned > +#endif > + > #define USE_AS_STPCPY > #define USE_AS_STRNCPY > -#define STRCPY __stpncpy_sse2_unaligned > +#define STRCPY STPNCPY > #include "strcpy-sse2-unaligned.S" > diff --git a/sysdeps/x86_64/multiarch/strcat-avx2.S b/sysdeps/x86_64/multiarch/strcat-avx2.S > index 51d42f89e8..d9b7fb2a43 100644 > --- a/sysdeps/x86_64/multiarch/strcat-avx2.S > +++ b/sysdeps/x86_64/multiarch/strcat-avx2.S > @@ -16,7 +16,10 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > +#include <isa-level.h> > + > +#if ISA_SHOULD_BUILD (3) > + > > # include <sysdep.h> > > diff --git a/sysdeps/x86_64/multiarch/strcat-evex.S b/sysdeps/x86_64/multiarch/strcat-evex.S > index 110505cb13..0e2df947e9 100644 > --- a/sysdeps/x86_64/multiarch/strcat-evex.S > +++ b/sysdeps/x86_64/multiarch/strcat-evex.S > @@ -16,7 +16,10 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > +#include <isa-level.h> > + > +#if ISA_SHOULD_BUILD (4) > + > > # include <sysdep.h> > > diff --git a/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S > index a0e051d08a..9d2ca1d504 100644 > --- a/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S > +++ b/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S > @@ -16,7 +16,12 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > +#include <isa-level.h> > + > +/* MINIMUM_X86_ISA_LEVEL <= 2 because there is no V2 implementation > + so we need this to build for ISA V2 builds. */ > +#if ISA_SHOULD_BUILD (2) > + > > # include <sysdep.h> > > diff --git a/sysdeps/x86_64/multiarch/strcat-sse2.S b/sysdeps/x86_64/multiarch/strcat-sse2.S > index 244c4a6d74..d1d0a3366a 100644 > --- a/sysdeps/x86_64/multiarch/strcat-sse2.S > +++ b/sysdeps/x86_64/multiarch/strcat-sse2.S > @@ -1,5 +1,6 @@ > -/* strcat optimized with SSE2. > - Copyright (C) 2017-2022 Free Software Foundation, Inc. > +/* strcat(dest, src) -- Append SRC on the end of DEST. > + Optimized for x86-64. > + Copyright (C) 2002-2022 Free Software Foundation, Inc. > This file is part of the GNU C Library. > > The GNU C Library is free software; you can redistribute it and/or > @@ -16,13 +17,17 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > +#include <isa-level.h> > + > +#if ISA_SHOULD_BUILD (1) > + > +# include <sysdep.h> > + > # ifndef STRCAT > -# define STRCAT __strcat_sse2 > +# define STRCAT __strcat_sse2 > # endif > -#endif > > -#include <sysdep.h> > +/* Will be removed when new strcpy implementation gets merged. */ > > .text > ENTRY (STRCAT) > @@ -256,3 +261,4 @@ ENTRY (STRCAT) > movq %rdi, %rax /* Source is return value. */ > retq > END (STRCAT) > +#endif > diff --git a/sysdeps/x86_64/multiarch/strcpy-avx2.S b/sysdeps/x86_64/multiarch/strcpy-avx2.S > index 064eafcbee..c725834929 100644 > --- a/sysdeps/x86_64/multiarch/strcpy-avx2.S > +++ b/sysdeps/x86_64/multiarch/strcpy-avx2.S > @@ -16,7 +16,10 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > +#include <isa-level.h> > + > +#if ISA_SHOULD_BUILD (3) > + > > # ifndef USE_AS_STRCAT > # include <sysdep.h> > diff --git a/sysdeps/x86_64/multiarch/strcpy-evex.S b/sysdeps/x86_64/multiarch/strcpy-evex.S > index 32229e05d8..82e45ac675 100644 > --- a/sysdeps/x86_64/multiarch/strcpy-evex.S > +++ b/sysdeps/x86_64/multiarch/strcpy-evex.S > @@ -16,7 +16,10 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > +#include <isa-level.h> > + > +#if ISA_SHOULD_BUILD (4) > + > > # ifndef USE_AS_STRCAT > # include <sysdep.h> > diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S > index d5fbd9570a..a889cd96be 100644 > --- a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S > +++ b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S > @@ -16,7 +16,12 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > +#include <isa-level.h> > + > +/* MINIMUM_X86_ISA_LEVEL <= 2 because there is no V2 implementation > + so we need this to build for ISA V2 builds. */ > +#if ISA_SHOULD_BUILD (2) > + > > # ifndef USE_AS_STRCAT > # include <sysdep.h> > diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2.S b/sysdeps/x86_64/multiarch/strcpy-sse2.S > index 8b5db8b13d..e29b411314 100644 > --- a/sysdeps/x86_64/multiarch/strcpy-sse2.S > +++ b/sysdeps/x86_64/multiarch/strcpy-sse2.S > @@ -1,5 +1,5 @@ > -/* strcpy optimized with SSE2. > - Copyright (C) 2017-2022 Free Software Foundation, Inc. > +/* strcpy/stpcpy implementation for x86-64. > + Copyright (C) 2002-2022 Free Software Foundation, Inc. > This file is part of the GNU C Library. > > The GNU C Library is free software; you can redistribute it and/or > @@ -16,13 +16,15 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#if IS_IN (libc) > -# ifndef STRCPY > +#include <isa-level.h> > + > +#if ISA_SHOULD_BUILD (1) > + > +# include <sysdep.h> > + > +# ifndef STPCPY > # define STRCPY __strcpy_sse2 > # endif > -#endif > - > -#include <sysdep.h> > > .text > ENTRY (STRCPY) > @@ -144,10 +146,11 @@ ENTRY (STRCPY) > jmp 3b /* and look at next two bytes in %rax. */ > > 4: > -#ifdef USE_AS_STPCPY > +# ifdef USE_AS_STPCPY > movq %rdx, %rax /* Destination is return value. */ > -#else > +# else > movq %rdi, %rax /* Source is return value. */ > -#endif > +# endif > retq > END (STRCPY) > +#endif > diff --git a/sysdeps/x86_64/multiarch/strncat-avx2.S b/sysdeps/x86_64/multiarch/strncat-avx2.S > index bfefa659bb..52ecbca943 100644 > --- a/sysdeps/x86_64/multiarch/strncat-avx2.S > +++ b/sysdeps/x86_64/multiarch/strncat-avx2.S > @@ -1,3 +1,7 @@ > +#ifndef STRNCAT > +# define STRNCAT __strncat_avx2 > +#endif > + > #define USE_AS_STRNCAT > -#define STRCAT __strncat_avx2 > +#define STRCAT STRNCAT > #include "strcat-avx2.S" > diff --git a/sysdeps/x86_64/multiarch/strncat-evex.S b/sysdeps/x86_64/multiarch/strncat-evex.S > index 8884f02371..203a19bf21 100644 > --- a/sysdeps/x86_64/multiarch/strncat-evex.S > +++ b/sysdeps/x86_64/multiarch/strncat-evex.S > @@ -1,3 +1,7 @@ > +#ifndef STRNCAT > +# define STRNCAT __strncat_evex > +#endif > + > #define USE_AS_STRNCAT > -#define STRCAT __strncat_evex > +#define STRCAT STRNCAT > #include "strcat-evex.S" > diff --git a/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S > index 133e1d20b0..5982e22677 100644 > --- a/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S > +++ b/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S > @@ -1,3 +1,7 @@ > +#ifndef STRNCAT > +# define STRNCAT __strncat_sse2_unaligned > +#endif > + > #define USE_AS_STRNCAT > -#define STRCAT __strncat_sse2_unaligned > +#define STRCAT STRNCAT > #include "strcat-sse2-unaligned.S" > diff --git a/sysdeps/x86_64/multiarch/strncpy-avx2.S b/sysdeps/x86_64/multiarch/strncpy-avx2.S > index 9ef8c87627..ce634e94fa 100644 > --- a/sysdeps/x86_64/multiarch/strncpy-avx2.S > +++ b/sysdeps/x86_64/multiarch/strncpy-avx2.S > @@ -1,3 +1,7 @@ > +#ifndef STRNCPY > +# define STRNCPY __strncpy_avx2 > +#endif > + > #define USE_AS_STRNCPY > -#define STRCPY __strncpy_avx2 > +#define STRCPY STRNCPY > #include "strcpy-avx2.S" > diff --git a/sysdeps/x86_64/multiarch/strncpy-evex.S b/sysdeps/x86_64/multiarch/strncpy-evex.S > index 40e391f0da..1b3426d511 100644 > --- a/sysdeps/x86_64/multiarch/strncpy-evex.S > +++ b/sysdeps/x86_64/multiarch/strncpy-evex.S > @@ -1,3 +1,7 @@ > +#ifndef STRNCPY > +# define STRNCPY __strncpy_evex > +#endif > + > #define USE_AS_STRNCPY > -#define STRCPY __strncpy_evex > +#define STRCPY STRNCPY > #include "strcpy-evex.S" > diff --git a/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S > index fcc23a754a..e3eb15d93a 100644 > --- a/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S > +++ b/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S > @@ -1,3 +1,7 @@ > +#ifndef STRNCPY > +# define STRNCPY __strncpy_sse2_unaligned > +#endif > + > #define USE_AS_STRNCPY > -#define STRCPY __strncpy_sse2_unaligned > +#define STRCPY STRNCPY > #include "strcpy-sse2-unaligned.S" > diff --git a/sysdeps/x86_64/stpcpy.S b/sysdeps/x86_64/stpcpy.S > index b097c203dd..c7d8d959a7 100644 > --- a/sysdeps/x86_64/stpcpy.S > +++ b/sysdeps/x86_64/stpcpy.S > @@ -1,6 +1,28 @@ > -#define STRCPY __stpcpy > +/* stpcpy dispatch for RTLD and non-multiarch build > + Copyright (C) 2022 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > > -#include "multiarch/stpcpy-sse2.S" > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#define STPCPY __stpcpy > + > +#define DEFAULT_IMPL_V1 "multiarch/stpcpy-sse2-unaligned.S" > +#define DEFAULT_IMPL_V3 "multiarch/stpcpy-avx2.S" > +#define DEFAULT_IMPL_V4 "multiarch/stpcpy-evex.S" > + > +#include "isa-default-impl.h" > > weak_alias (__stpcpy, stpcpy) > libc_hidden_def (__stpcpy) > diff --git a/sysdeps/x86_64/stpncpy.S b/sysdeps/x86_64/stpncpy.S > new file mode 100644 > index 0000000000..5c2d6efa3d > --- /dev/null > +++ b/sysdeps/x86_64/stpncpy.S > @@ -0,0 +1,28 @@ > +/* stpncpy dispatch for RTLD and non-multiarch build > + Copyright (C) 2022 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#define STPNCPY __stpncpy > + > +#define DEFAULT_IMPL_V1 "multiarch/stpncpy-sse2-unaligned.S" > +#define DEFAULT_IMPL_V3 "multiarch/stpncpy-avx2.S" > +#define DEFAULT_IMPL_V4 "multiarch/stpncpy-evex.S" > + > +#include "isa-default-impl.h" > + > +weak_alias (__stpncpy, stpncpy) > +libc_hidden_def (__stpncpy) > diff --git a/sysdeps/x86_64/strcat.S b/sysdeps/x86_64/strcat.S > index fc3e8a9bcf..9bca7ecedc 100644 > --- a/sysdeps/x86_64/strcat.S > +++ b/sysdeps/x86_64/strcat.S > @@ -1,6 +1,5 @@ > -/* strcat(dest, src) -- Append SRC on the end of DEST. > - Optimized for x86-64. > - Copyright (C) 2002-2022 Free Software Foundation, Inc. > +/* strcat dispatch for RTLD and non-multiarch build > + Copyright (C) 2022 Free Software Foundation, Inc. > This file is part of the GNU C Library. > > The GNU C Library is free software; you can redistribute it and/or > @@ -17,6 +16,12 @@ > License along with the GNU C Library; if not, see > <https://www.gnu.org/licenses/>. */ > > -#define STRCAT strcat > -#include "multiarch/strcat-sse2.S" > +#define STRCAT strcat > + > +#define DEFAULT_IMPL_V1 "multiarch/strcat-sse2-unaligned.S" > +#define DEFAULT_IMPL_V3 "multiarch/strcat-avx2.S" > +#define DEFAULT_IMPL_V4 "multiarch/strcat-evex.S" > + > +#include "isa-default-impl.h" > + > libc_hidden_builtin_def (strcat) > diff --git a/sysdeps/x86_64/strcpy.S b/sysdeps/x86_64/strcpy.S > index 05f19e6e94..7c04cc2102 100644 > --- a/sysdeps/x86_64/strcpy.S > +++ b/sysdeps/x86_64/strcpy.S > @@ -1,5 +1,5 @@ > -/* strcpy/stpcpy implementation for x86-64. > - Copyright (C) 2002-2022 Free Software Foundation, Inc. > +/* strcpy dispatch for RTLD and non-multiarch build > + Copyright (C) 2022 Free Software Foundation, Inc. > This file is part of the GNU C Library. > > The GNU C Library is free software; you can redistribute it and/or > @@ -17,5 +17,11 @@ > <https://www.gnu.org/licenses/>. */ > > #define STRCPY strcpy > -#include "multiarch/strcpy-sse2.S" > + > +#define DEFAULT_IMPL_V1 "multiarch/strcpy-sse2-unaligned.S" > +#define DEFAULT_IMPL_V3 "multiarch/strcpy-avx2.S" > +#define DEFAULT_IMPL_V4 "multiarch/strcpy-evex.S" > + > +#include "isa-default-impl.h" > + > libc_hidden_builtin_def (strcpy) > diff --git a/sysdeps/x86_64/strncat.S b/sysdeps/x86_64/strncat.S > new file mode 100644 > index 0000000000..3ba2603a28 > --- /dev/null > +++ b/sysdeps/x86_64/strncat.S > @@ -0,0 +1,28 @@ > +/* strncat dispatch for RTLD and non-multiarch build > + Copyright (C) 2022 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#define STRNCAT strncat > + > +#define DEFAULT_IMPL_V1 "multiarch/strncat-sse2-unaligned.S" > +#define DEFAULT_IMPL_V3 "multiarch/strncat-avx2.S" > +#define DEFAULT_IMPL_V4 "multiarch/strncat-evex.S" > + > +#include "isa-default-impl.h" > + > +strong_alias (strncat, __strncat) > +libc_hidden_def (__strncat) > diff --git a/sysdeps/x86_64/strncpy.S b/sysdeps/x86_64/strncpy.S > new file mode 100644 > index 0000000000..04c904e60d > --- /dev/null > +++ b/sysdeps/x86_64/strncpy.S > @@ -0,0 +1,27 @@ > +/* strncpy dispatch for RTLD and non-multiarch build > + Copyright (C) 2022 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#define STRNCPY strncpy > + > +#define DEFAULT_IMPL_V1 "multiarch/strncpy-sse2-unaligned.S" > +#define DEFAULT_IMPL_V3 "multiarch/strncpy-avx2.S" > +#define DEFAULT_IMPL_V4 "multiarch/strncpy-evex.S" > + > +#include "isa-default-impl.h" > + > +libc_hidden_builtin_def (strncpy) > -- > 2.34.1 > LGTM. Thanks.
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c index 9318e98cc8..a71444eccb 100644 --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c @@ -403,33 +403,46 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, /* Support sysdeps/x86_64/multiarch/stpncpy.c. */ IFUNC_IMPL (i, name, stpncpy, - IFUNC_IMPL_ADD (array, i, stpncpy, CPU_FEATURE_USABLE (AVX2), - __stpncpy_avx2) - IFUNC_IMPL_ADD (array, i, stpncpy, - (CPU_FEATURE_USABLE (AVX2) - && CPU_FEATURE_USABLE (RTM)), - __stpncpy_avx2_rtm) - IFUNC_IMPL_ADD (array, i, stpncpy, - (CPU_FEATURE_USABLE (AVX512VL) - && CPU_FEATURE_USABLE (AVX512BW)), - __stpncpy_evex) - IFUNC_IMPL_ADD (array, i, stpncpy, 1, - __stpncpy_sse2_unaligned)) + X86_IFUNC_IMPL_ADD_V4 (array, i, stpncpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), + __stpncpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, stpncpy, + CPU_FEATURE_USABLE (AVX2), + __stpncpy_avx2) + X86_IFUNC_IMPL_ADD_V3 (array, i, stpncpy, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (RTM)), + __stpncpy_avx2_rtm) + /* ISA V2 wrapper for sse2_unaligned implementation because + the sse2_unaligned implementation is also used at ISA + level 2. */ + X86_IFUNC_IMPL_ADD_V2 (array, i, stpncpy, + 1, + __stpncpy_sse2_unaligned)) /* Support sysdeps/x86_64/multiarch/stpcpy.c. */ IFUNC_IMPL (i, name, stpcpy, - IFUNC_IMPL_ADD (array, i, stpcpy, CPU_FEATURE_USABLE (AVX2), - __stpcpy_avx2) - IFUNC_IMPL_ADD (array, i, stpcpy, - (CPU_FEATURE_USABLE (AVX2) - && CPU_FEATURE_USABLE (RTM)), - __stpcpy_avx2_rtm) - IFUNC_IMPL_ADD (array, i, stpcpy, - (CPU_FEATURE_USABLE (AVX512VL) - && CPU_FEATURE_USABLE (AVX512BW)), - __stpcpy_evex) - IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2_unaligned) - IFUNC_IMPL_ADD (array, i, stpcpy, 1, __stpcpy_sse2)) + X86_IFUNC_IMPL_ADD_V4 (array, i, stpcpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), + __stpcpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, stpcpy, + CPU_FEATURE_USABLE (AVX2), + __stpcpy_avx2) + X86_IFUNC_IMPL_ADD_V3 (array, i, stpcpy, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (RTM)), + __stpcpy_avx2_rtm) + /* ISA V2 wrapper for sse2_unaligned implementation because + the sse2_unaligned implementation is also used at ISA + level 2. */ + X86_IFUNC_IMPL_ADD_V2 (array, i, stpcpy, + 1, + __stpcpy_sse2_unaligned) + X86_IFUNC_IMPL_ADD_V1 (array, i, stpcpy, + 1, + __stpcpy_sse2)) /* Support sysdeps/x86_64/multiarch/strcasecmp_l.c. */ IFUNC_IMPL (i, name, strcasecmp, @@ -477,18 +490,26 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, /* Support sysdeps/x86_64/multiarch/strcat.c. */ IFUNC_IMPL (i, name, strcat, - IFUNC_IMPL_ADD (array, i, strcat, CPU_FEATURE_USABLE (AVX2), - __strcat_avx2) - IFUNC_IMPL_ADD (array, i, strcat, - (CPU_FEATURE_USABLE (AVX2) - && CPU_FEATURE_USABLE (RTM)), - __strcat_avx2_rtm) - IFUNC_IMPL_ADD (array, i, strcat, - (CPU_FEATURE_USABLE (AVX512VL) - && CPU_FEATURE_USABLE (AVX512BW)), - __strcat_evex) - IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2_unaligned) - IFUNC_IMPL_ADD (array, i, strcat, 1, __strcat_sse2)) + X86_IFUNC_IMPL_ADD_V4 (array, i, strcat, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), + __strcat_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, strcat, + CPU_FEATURE_USABLE (AVX2), + __strcat_avx2) + X86_IFUNC_IMPL_ADD_V3 (array, i, strcat, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (RTM)), + __strcat_avx2_rtm) + /* ISA V2 wrapper for sse2_unaligned implementation because + the sse2_unaligned implementation is also used at ISA + level 2. */ + X86_IFUNC_IMPL_ADD_V2 (array, i, strcat, + 1, + __strcat_sse2_unaligned) + X86_IFUNC_IMPL_ADD_V1 (array, i, strcat, + 1, + __strcat_sse2)) /* Support sysdeps/x86_64/multiarch/strchr.c. */ IFUNC_IMPL (i, name, strchr, @@ -584,18 +605,26 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, /* Support sysdeps/x86_64/multiarch/strcpy.c. */ IFUNC_IMPL (i, name, strcpy, - IFUNC_IMPL_ADD (array, i, strcpy, CPU_FEATURE_USABLE (AVX2), - __strcpy_avx2) - IFUNC_IMPL_ADD (array, i, strcpy, - (CPU_FEATURE_USABLE (AVX2) - && CPU_FEATURE_USABLE (RTM)), - __strcpy_avx2_rtm) - IFUNC_IMPL_ADD (array, i, strcpy, - (CPU_FEATURE_USABLE (AVX512VL) - && CPU_FEATURE_USABLE (AVX512BW)), - __strcpy_evex) - IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2_unaligned) - IFUNC_IMPL_ADD (array, i, strcpy, 1, __strcpy_sse2)) + X86_IFUNC_IMPL_ADD_V4 (array, i, strcpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), + __strcpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, strcpy, + CPU_FEATURE_USABLE (AVX2), + __strcpy_avx2) + X86_IFUNC_IMPL_ADD_V3 (array, i, strcpy, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (RTM)), + __strcpy_avx2_rtm) + /* ISA V2 wrapper for sse2_unaligned implementation because + the sse2_unaligned implementation is also used at ISA + level 2. */ + X86_IFUNC_IMPL_ADD_V2 (array, i, strcpy, + 1, + __strcpy_sse2_unaligned) + X86_IFUNC_IMPL_ADD_V1 (array, i, strcpy, + 1, + __strcpy_sse2)) /* Support sysdeps/x86_64/multiarch/strcspn.c. */ IFUNC_IMPL (i, name, strcspn, @@ -651,33 +680,43 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, /* Support sysdeps/x86_64/multiarch/strncat.c. */ IFUNC_IMPL (i, name, strncat, - IFUNC_IMPL_ADD (array, i, strncat, CPU_FEATURE_USABLE (AVX2), - __strncat_avx2) - IFUNC_IMPL_ADD (array, i, strncat, - (CPU_FEATURE_USABLE (AVX2) - && CPU_FEATURE_USABLE (RTM)), - __strncat_avx2_rtm) - IFUNC_IMPL_ADD (array, i, strncat, - (CPU_FEATURE_USABLE (AVX512VL) - && CPU_FEATURE_USABLE (AVX512BW)), - __strncat_evex) - IFUNC_IMPL_ADD (array, i, strncat, 1, - __strncat_sse2_unaligned)) + X86_IFUNC_IMPL_ADD_V4 (array, i, strncat, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), + __strncat_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, strncat, + CPU_FEATURE_USABLE (AVX2), + __strncat_avx2) + X86_IFUNC_IMPL_ADD_V3 (array, i, strncat, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (RTM)), + __strncat_avx2_rtm) + /* ISA V2 wrapper for sse2_unaligned implementation because + the sse2_unaligned implementation is also used at ISA + level 2. */ + X86_IFUNC_IMPL_ADD_V2 (array, i, strncat, + 1, + __strncat_sse2_unaligned)) /* Support sysdeps/x86_64/multiarch/strncpy.c. */ IFUNC_IMPL (i, name, strncpy, - IFUNC_IMPL_ADD (array, i, strncpy, CPU_FEATURE_USABLE (AVX2), - __strncpy_avx2) - IFUNC_IMPL_ADD (array, i, strncpy, - (CPU_FEATURE_USABLE (AVX2) - && CPU_FEATURE_USABLE (RTM)), - __strncpy_avx2_rtm) - IFUNC_IMPL_ADD (array, i, strncpy, - (CPU_FEATURE_USABLE (AVX512VL) - && CPU_FEATURE_USABLE (AVX512BW)), - __strncpy_evex) - IFUNC_IMPL_ADD (array, i, strncpy, 1, - __strncpy_sse2_unaligned)) + X86_IFUNC_IMPL_ADD_V4 (array, i, strncpy, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW)), + __strncpy_evex) + X86_IFUNC_IMPL_ADD_V3 (array, i, strncpy, + CPU_FEATURE_USABLE (AVX2), + __strncpy_avx2) + X86_IFUNC_IMPL_ADD_V3 (array, i, strncpy, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (RTM)), + __strncpy_avx2_rtm) + /* ISA V2 wrapper for sse2_unaligned implementation because + the sse2_unaligned implementation is also used at ISA + level 2. */ + X86_IFUNC_IMPL_ADD_V2 (array, i, strncpy, + 1, + __strncpy_sse2_unaligned)) /* Support sysdeps/x86_64/multiarch/strpbrk.c. */ IFUNC_IMPL (i, name, strpbrk, diff --git a/sysdeps/x86_64/multiarch/ifunc-strcpy.h b/sysdeps/x86_64/multiarch/ifunc-strcpy.h index a15afa44e9..e083f71df3 100644 --- a/sysdeps/x86_64/multiarch/ifunc-strcpy.h +++ b/sysdeps/x86_64/multiarch/ifunc-strcpy.h @@ -20,33 +20,38 @@ #include <init-arch.h> -extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; -extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) - attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + +extern __typeof (REDIRECT_NAME) + OPTIMIZE (sse2_unaligned) attribute_hidden; + +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; static inline void * IFUNC_SELECTOR (void) { - const struct cpu_features* cpu_features = __get_cpu_features (); + const struct cpu_features *cpu_features = __get_cpu_features (); - if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) - && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, + AVX_Fast_Unaligned_Load, )) { - if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) - && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) return OPTIMIZE (evex); if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) return OPTIMIZE (avx2_rtm); - if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, + Prefer_No_VZEROUPPER, !)) return OPTIMIZE (avx2); } - if (CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load)) + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Fast_Unaligned_Load, )) return OPTIMIZE (sse2_unaligned); return OPTIMIZE (sse2); diff --git a/sysdeps/x86_64/multiarch/ifunc-strncpy.h b/sysdeps/x86_64/multiarch/ifunc-strncpy.h index 323225af4d..5c1c46b885 100644 --- a/sysdeps/x86_64/multiarch/ifunc-strncpy.h +++ b/sysdeps/x86_64/multiarch/ifunc-strncpy.h @@ -19,28 +19,32 @@ #include <init-arch.h> -extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2_unaligned) - attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden; extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden; -extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden; + +extern __typeof (REDIRECT_NAME) + OPTIMIZE (sse2_unaligned) attribute_hidden; static inline void * IFUNC_SELECTOR (void) { - const struct cpu_features* cpu_features = __get_cpu_features (); + const struct cpu_features *cpu_features = __get_cpu_features (); - if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) - && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, + AVX_Fast_Unaligned_Load, )) { - if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) - && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) + && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) return OPTIMIZE (evex); if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) return OPTIMIZE (avx2_rtm); - if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, + Prefer_No_VZEROUPPER, !)) return OPTIMIZE (avx2); } diff --git a/sysdeps/x86_64/multiarch/stpcpy-avx2.S b/sysdeps/x86_64/multiarch/stpcpy-avx2.S index f0bd3029fe..277f9e73d2 100644 --- a/sysdeps/x86_64/multiarch/stpcpy-avx2.S +++ b/sysdeps/x86_64/multiarch/stpcpy-avx2.S @@ -1,3 +1,7 @@ +#ifndef STPCPY +# define STPCPY __stpcpy_avx2 +#endif + #define USE_AS_STPCPY -#define STRCPY __stpcpy_avx2 +#define STRCPY STPCPY #include "strcpy-avx2.S" diff --git a/sysdeps/x86_64/multiarch/stpcpy-evex.S b/sysdeps/x86_64/multiarch/stpcpy-evex.S index 7c6f26cd98..4f1c015424 100644 --- a/sysdeps/x86_64/multiarch/stpcpy-evex.S +++ b/sysdeps/x86_64/multiarch/stpcpy-evex.S @@ -1,3 +1,7 @@ +#ifndef STPCPY +# define STPCPY __stpcpy_evex +#endif + #define USE_AS_STPCPY -#define STRCPY __stpcpy_evex +#define STRCPY STPCPY #include "strcpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S index 34231f8b46..4c77e5b51c 100644 --- a/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S +++ b/sysdeps/x86_64/multiarch/stpcpy-sse2-unaligned.S @@ -1,3 +1,7 @@ +#ifndef STPCPY +# define STPCPY __stpcpy_sse2_unaligned +#endif + #define USE_AS_STPCPY -#define STRCPY __stpcpy_sse2_unaligned +#define STRCPY STPCPY #include "strcpy-sse2-unaligned.S" diff --git a/sysdeps/x86_64/multiarch/stpcpy-sse2.S b/sysdeps/x86_64/multiarch/stpcpy-sse2.S index ea9f973af3..fcd67cada2 100644 --- a/sysdeps/x86_64/multiarch/stpcpy-sse2.S +++ b/sysdeps/x86_64/multiarch/stpcpy-sse2.S @@ -1,26 +1,7 @@ -/* stpcpy optimized with SSE2. - Copyright (C) 2017-2022 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#if IS_IN (libc) -# ifndef STRCPY -# define STRCPY __stpcpy_sse2 -# endif +#ifndef STPCPY +# define STPCPY __stpcpy_sse2 #endif #define USE_AS_STPCPY +#define STRCPY STPCPY #include "strcpy-sse2.S" diff --git a/sysdeps/x86_64/multiarch/stpncpy-avx2.S b/sysdeps/x86_64/multiarch/stpncpy-avx2.S index 032b0407d0..b2f8c19143 100644 --- a/sysdeps/x86_64/multiarch/stpncpy-avx2.S +++ b/sysdeps/x86_64/multiarch/stpncpy-avx2.S @@ -1,4 +1,8 @@ +#ifndef STPNCPY +# define STPNCPY __stpncpy_avx2 +#endif + #define USE_AS_STPCPY #define USE_AS_STRNCPY -#define STRCPY __stpncpy_avx2 +#define STRCPY STPNCPY #include "strcpy-avx2.S" diff --git a/sysdeps/x86_64/multiarch/stpncpy-evex.S b/sysdeps/x86_64/multiarch/stpncpy-evex.S index 1570014d1c..99ea76a372 100644 --- a/sysdeps/x86_64/multiarch/stpncpy-evex.S +++ b/sysdeps/x86_64/multiarch/stpncpy-evex.S @@ -1,4 +1,8 @@ +#ifndef STPNCPY +# define STPNCPY __stpncpy_evex +#endif + #define USE_AS_STPCPY #define USE_AS_STRNCPY -#define STRCPY __stpncpy_evex +#define STRCPY STPNCPY #include "strcpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S index 658520f78f..c83706016f 100644 --- a/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S +++ b/sysdeps/x86_64/multiarch/stpncpy-sse2-unaligned.S @@ -1,4 +1,8 @@ +#ifndef STPNCPY +# define STPNCPY __stpncpy_sse2_unaligned +#endif + #define USE_AS_STPCPY #define USE_AS_STRNCPY -#define STRCPY __stpncpy_sse2_unaligned +#define STRCPY STPNCPY #include "strcpy-sse2-unaligned.S" diff --git a/sysdeps/x86_64/multiarch/strcat-avx2.S b/sysdeps/x86_64/multiarch/strcat-avx2.S index 51d42f89e8..d9b7fb2a43 100644 --- a/sysdeps/x86_64/multiarch/strcat-avx2.S +++ b/sysdeps/x86_64/multiarch/strcat-avx2.S @@ -16,7 +16,10 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) +#include <isa-level.h> + +#if ISA_SHOULD_BUILD (3) + # include <sysdep.h> diff --git a/sysdeps/x86_64/multiarch/strcat-evex.S b/sysdeps/x86_64/multiarch/strcat-evex.S index 110505cb13..0e2df947e9 100644 --- a/sysdeps/x86_64/multiarch/strcat-evex.S +++ b/sysdeps/x86_64/multiarch/strcat-evex.S @@ -16,7 +16,10 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) +#include <isa-level.h> + +#if ISA_SHOULD_BUILD (4) + # include <sysdep.h> diff --git a/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S index a0e051d08a..9d2ca1d504 100644 --- a/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S +++ b/sysdeps/x86_64/multiarch/strcat-sse2-unaligned.S @@ -16,7 +16,12 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) +#include <isa-level.h> + +/* MINIMUM_X86_ISA_LEVEL <= 2 because there is no V2 implementation + so we need this to build for ISA V2 builds. */ +#if ISA_SHOULD_BUILD (2) + # include <sysdep.h> diff --git a/sysdeps/x86_64/multiarch/strcat-sse2.S b/sysdeps/x86_64/multiarch/strcat-sse2.S index 244c4a6d74..d1d0a3366a 100644 --- a/sysdeps/x86_64/multiarch/strcat-sse2.S +++ b/sysdeps/x86_64/multiarch/strcat-sse2.S @@ -1,5 +1,6 @@ -/* strcat optimized with SSE2. - Copyright (C) 2017-2022 Free Software Foundation, Inc. +/* strcat(dest, src) -- Append SRC on the end of DEST. + Optimized for x86-64. + Copyright (C) 2002-2022 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -16,13 +17,17 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) +#include <isa-level.h> + +#if ISA_SHOULD_BUILD (1) + +# include <sysdep.h> + # ifndef STRCAT -# define STRCAT __strcat_sse2 +# define STRCAT __strcat_sse2 # endif -#endif -#include <sysdep.h> +/* Will be removed when new strcpy implementation gets merged. */ .text ENTRY (STRCAT) @@ -256,3 +261,4 @@ ENTRY (STRCAT) movq %rdi, %rax /* Source is return value. */ retq END (STRCAT) +#endif diff --git a/sysdeps/x86_64/multiarch/strcpy-avx2.S b/sysdeps/x86_64/multiarch/strcpy-avx2.S index 064eafcbee..c725834929 100644 --- a/sysdeps/x86_64/multiarch/strcpy-avx2.S +++ b/sysdeps/x86_64/multiarch/strcpy-avx2.S @@ -16,7 +16,10 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) +#include <isa-level.h> + +#if ISA_SHOULD_BUILD (3) + # ifndef USE_AS_STRCAT # include <sysdep.h> diff --git a/sysdeps/x86_64/multiarch/strcpy-evex.S b/sysdeps/x86_64/multiarch/strcpy-evex.S index 32229e05d8..82e45ac675 100644 --- a/sysdeps/x86_64/multiarch/strcpy-evex.S +++ b/sysdeps/x86_64/multiarch/strcpy-evex.S @@ -16,7 +16,10 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) +#include <isa-level.h> + +#if ISA_SHOULD_BUILD (4) + # ifndef USE_AS_STRCAT # include <sysdep.h> diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S index d5fbd9570a..a889cd96be 100644 --- a/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S +++ b/sysdeps/x86_64/multiarch/strcpy-sse2-unaligned.S @@ -16,7 +16,12 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) +#include <isa-level.h> + +/* MINIMUM_X86_ISA_LEVEL <= 2 because there is no V2 implementation + so we need this to build for ISA V2 builds. */ +#if ISA_SHOULD_BUILD (2) + # ifndef USE_AS_STRCAT # include <sysdep.h> diff --git a/sysdeps/x86_64/multiarch/strcpy-sse2.S b/sysdeps/x86_64/multiarch/strcpy-sse2.S index 8b5db8b13d..e29b411314 100644 --- a/sysdeps/x86_64/multiarch/strcpy-sse2.S +++ b/sysdeps/x86_64/multiarch/strcpy-sse2.S @@ -1,5 +1,5 @@ -/* strcpy optimized with SSE2. - Copyright (C) 2017-2022 Free Software Foundation, Inc. +/* strcpy/stpcpy implementation for x86-64. + Copyright (C) 2002-2022 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -16,13 +16,15 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#if IS_IN (libc) -# ifndef STRCPY +#include <isa-level.h> + +#if ISA_SHOULD_BUILD (1) + +# include <sysdep.h> + +# ifndef STPCPY # define STRCPY __strcpy_sse2 # endif -#endif - -#include <sysdep.h> .text ENTRY (STRCPY) @@ -144,10 +146,11 @@ ENTRY (STRCPY) jmp 3b /* and look at next two bytes in %rax. */ 4: -#ifdef USE_AS_STPCPY +# ifdef USE_AS_STPCPY movq %rdx, %rax /* Destination is return value. */ -#else +# else movq %rdi, %rax /* Source is return value. */ -#endif +# endif retq END (STRCPY) +#endif diff --git a/sysdeps/x86_64/multiarch/strncat-avx2.S b/sysdeps/x86_64/multiarch/strncat-avx2.S index bfefa659bb..52ecbca943 100644 --- a/sysdeps/x86_64/multiarch/strncat-avx2.S +++ b/sysdeps/x86_64/multiarch/strncat-avx2.S @@ -1,3 +1,7 @@ +#ifndef STRNCAT +# define STRNCAT __strncat_avx2 +#endif + #define USE_AS_STRNCAT -#define STRCAT __strncat_avx2 +#define STRCAT STRNCAT #include "strcat-avx2.S" diff --git a/sysdeps/x86_64/multiarch/strncat-evex.S b/sysdeps/x86_64/multiarch/strncat-evex.S index 8884f02371..203a19bf21 100644 --- a/sysdeps/x86_64/multiarch/strncat-evex.S +++ b/sysdeps/x86_64/multiarch/strncat-evex.S @@ -1,3 +1,7 @@ +#ifndef STRNCAT +# define STRNCAT __strncat_evex +#endif + #define USE_AS_STRNCAT -#define STRCAT __strncat_evex +#define STRCAT STRNCAT #include "strcat-evex.S" diff --git a/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S index 133e1d20b0..5982e22677 100644 --- a/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S +++ b/sysdeps/x86_64/multiarch/strncat-sse2-unaligned.S @@ -1,3 +1,7 @@ +#ifndef STRNCAT +# define STRNCAT __strncat_sse2_unaligned +#endif + #define USE_AS_STRNCAT -#define STRCAT __strncat_sse2_unaligned +#define STRCAT STRNCAT #include "strcat-sse2-unaligned.S" diff --git a/sysdeps/x86_64/multiarch/strncpy-avx2.S b/sysdeps/x86_64/multiarch/strncpy-avx2.S index 9ef8c87627..ce634e94fa 100644 --- a/sysdeps/x86_64/multiarch/strncpy-avx2.S +++ b/sysdeps/x86_64/multiarch/strncpy-avx2.S @@ -1,3 +1,7 @@ +#ifndef STRNCPY +# define STRNCPY __strncpy_avx2 +#endif + #define USE_AS_STRNCPY -#define STRCPY __strncpy_avx2 +#define STRCPY STRNCPY #include "strcpy-avx2.S" diff --git a/sysdeps/x86_64/multiarch/strncpy-evex.S b/sysdeps/x86_64/multiarch/strncpy-evex.S index 40e391f0da..1b3426d511 100644 --- a/sysdeps/x86_64/multiarch/strncpy-evex.S +++ b/sysdeps/x86_64/multiarch/strncpy-evex.S @@ -1,3 +1,7 @@ +#ifndef STRNCPY +# define STRNCPY __strncpy_evex +#endif + #define USE_AS_STRNCPY -#define STRCPY __strncpy_evex +#define STRCPY STRNCPY #include "strcpy-evex.S" diff --git a/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S b/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S index fcc23a754a..e3eb15d93a 100644 --- a/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S +++ b/sysdeps/x86_64/multiarch/strncpy-sse2-unaligned.S @@ -1,3 +1,7 @@ +#ifndef STRNCPY +# define STRNCPY __strncpy_sse2_unaligned +#endif + #define USE_AS_STRNCPY -#define STRCPY __strncpy_sse2_unaligned +#define STRCPY STRNCPY #include "strcpy-sse2-unaligned.S" diff --git a/sysdeps/x86_64/stpcpy.S b/sysdeps/x86_64/stpcpy.S index b097c203dd..c7d8d959a7 100644 --- a/sysdeps/x86_64/stpcpy.S +++ b/sysdeps/x86_64/stpcpy.S @@ -1,6 +1,28 @@ -#define STRCPY __stpcpy +/* stpcpy dispatch for RTLD and non-multiarch build + Copyright (C) 2022 Free Software Foundation, Inc. + This file is part of the GNU C Library. -#include "multiarch/stpcpy-sse2.S" + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#define STPCPY __stpcpy + +#define DEFAULT_IMPL_V1 "multiarch/stpcpy-sse2-unaligned.S" +#define DEFAULT_IMPL_V3 "multiarch/stpcpy-avx2.S" +#define DEFAULT_IMPL_V4 "multiarch/stpcpy-evex.S" + +#include "isa-default-impl.h" weak_alias (__stpcpy, stpcpy) libc_hidden_def (__stpcpy) diff --git a/sysdeps/x86_64/stpncpy.S b/sysdeps/x86_64/stpncpy.S new file mode 100644 index 0000000000..5c2d6efa3d --- /dev/null +++ b/sysdeps/x86_64/stpncpy.S @@ -0,0 +1,28 @@ +/* stpncpy dispatch for RTLD and non-multiarch build + Copyright (C) 2022 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#define STPNCPY __stpncpy + +#define DEFAULT_IMPL_V1 "multiarch/stpncpy-sse2-unaligned.S" +#define DEFAULT_IMPL_V3 "multiarch/stpncpy-avx2.S" +#define DEFAULT_IMPL_V4 "multiarch/stpncpy-evex.S" + +#include "isa-default-impl.h" + +weak_alias (__stpncpy, stpncpy) +libc_hidden_def (__stpncpy) diff --git a/sysdeps/x86_64/strcat.S b/sysdeps/x86_64/strcat.S index fc3e8a9bcf..9bca7ecedc 100644 --- a/sysdeps/x86_64/strcat.S +++ b/sysdeps/x86_64/strcat.S @@ -1,6 +1,5 @@ -/* strcat(dest, src) -- Append SRC on the end of DEST. - Optimized for x86-64. - Copyright (C) 2002-2022 Free Software Foundation, Inc. +/* strcat dispatch for RTLD and non-multiarch build + Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -17,6 +16,12 @@ License along with the GNU C Library; if not, see <https://www.gnu.org/licenses/>. */ -#define STRCAT strcat -#include "multiarch/strcat-sse2.S" +#define STRCAT strcat + +#define DEFAULT_IMPL_V1 "multiarch/strcat-sse2-unaligned.S" +#define DEFAULT_IMPL_V3 "multiarch/strcat-avx2.S" +#define DEFAULT_IMPL_V4 "multiarch/strcat-evex.S" + +#include "isa-default-impl.h" + libc_hidden_builtin_def (strcat) diff --git a/sysdeps/x86_64/strcpy.S b/sysdeps/x86_64/strcpy.S index 05f19e6e94..7c04cc2102 100644 --- a/sysdeps/x86_64/strcpy.S +++ b/sysdeps/x86_64/strcpy.S @@ -1,5 +1,5 @@ -/* strcpy/stpcpy implementation for x86-64. - Copyright (C) 2002-2022 Free Software Foundation, Inc. +/* strcpy dispatch for RTLD and non-multiarch build + Copyright (C) 2022 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -17,5 +17,11 @@ <https://www.gnu.org/licenses/>. */ #define STRCPY strcpy -#include "multiarch/strcpy-sse2.S" + +#define DEFAULT_IMPL_V1 "multiarch/strcpy-sse2-unaligned.S" +#define DEFAULT_IMPL_V3 "multiarch/strcpy-avx2.S" +#define DEFAULT_IMPL_V4 "multiarch/strcpy-evex.S" + +#include "isa-default-impl.h" + libc_hidden_builtin_def (strcpy) diff --git a/sysdeps/x86_64/strncat.S b/sysdeps/x86_64/strncat.S new file mode 100644 index 0000000000..3ba2603a28 --- /dev/null +++ b/sysdeps/x86_64/strncat.S @@ -0,0 +1,28 @@ +/* strncat dispatch for RTLD and non-multiarch build + Copyright (C) 2022 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#define STRNCAT strncat + +#define DEFAULT_IMPL_V1 "multiarch/strncat-sse2-unaligned.S" +#define DEFAULT_IMPL_V3 "multiarch/strncat-avx2.S" +#define DEFAULT_IMPL_V4 "multiarch/strncat-evex.S" + +#include "isa-default-impl.h" + +strong_alias (strncat, __strncat) +libc_hidden_def (__strncat) diff --git a/sysdeps/x86_64/strncpy.S b/sysdeps/x86_64/strncpy.S new file mode 100644 index 0000000000..04c904e60d --- /dev/null +++ b/sysdeps/x86_64/strncpy.S @@ -0,0 +1,27 @@ +/* strncpy dispatch for RTLD and non-multiarch build + Copyright (C) 2022 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#define STRNCPY strncpy + +#define DEFAULT_IMPL_V1 "multiarch/strncpy-sse2-unaligned.S" +#define DEFAULT_IMPL_V3 "multiarch/strncpy-avx2.S" +#define DEFAULT_IMPL_V4 "multiarch/strncpy-evex.S" + +#include "isa-default-impl.h" + +libc_hidden_builtin_def (strncpy)
1. Add default ISA level selection in non-multiarch/rtld implementations. 2. Add ISA level build guards to different implementations. - I.e strcpy-avx2.S which is ISA level 3 will only build if compiled ISA level <= 3. Otherwise there is no reason to include it as we will always use one of the ISA level 4 implementations (strcpy-evex.S). 3. Refactor the ifunc selector and ifunc implementation list to use the ISA level aware wrapper macros that allow functions below the compiled ISA level (with a guranteed replacement) to be skipped. Tested with and without multiarch on x86_64 for ISA levels: {generic, x86-64-v2, x86-64-v3, x86-64-v4} And m32 with and without multiarch. --- sysdeps/x86_64/multiarch/ifunc-impl-list.c | 183 +++++++++++------- sysdeps/x86_64/multiarch/ifunc-strcpy.h | 27 +-- sysdeps/x86_64/multiarch/ifunc-strncpy.h | 22 ++- sysdeps/x86_64/multiarch/stpcpy-avx2.S | 6 +- sysdeps/x86_64/multiarch/stpcpy-evex.S | 6 +- .../x86_64/multiarch/stpcpy-sse2-unaligned.S | 6 +- sysdeps/x86_64/multiarch/stpcpy-sse2.S | 25 +-- sysdeps/x86_64/multiarch/stpncpy-avx2.S | 6 +- sysdeps/x86_64/multiarch/stpncpy-evex.S | 6 +- .../x86_64/multiarch/stpncpy-sse2-unaligned.S | 6 +- sysdeps/x86_64/multiarch/strcat-avx2.S | 5 +- sysdeps/x86_64/multiarch/strcat-evex.S | 5 +- .../x86_64/multiarch/strcat-sse2-unaligned.S | 7 +- sysdeps/x86_64/multiarch/strcat-sse2.S | 18 +- sysdeps/x86_64/multiarch/strcpy-avx2.S | 5 +- sysdeps/x86_64/multiarch/strcpy-evex.S | 5 +- .../x86_64/multiarch/strcpy-sse2-unaligned.S | 7 +- sysdeps/x86_64/multiarch/strcpy-sse2.S | 23 ++- sysdeps/x86_64/multiarch/strncat-avx2.S | 6 +- sysdeps/x86_64/multiarch/strncat-evex.S | 6 +- .../x86_64/multiarch/strncat-sse2-unaligned.S | 6 +- sysdeps/x86_64/multiarch/strncpy-avx2.S | 6 +- sysdeps/x86_64/multiarch/strncpy-evex.S | 6 +- .../x86_64/multiarch/strncpy-sse2-unaligned.S | 6 +- sysdeps/x86_64/stpcpy.S | 26 ++- sysdeps/x86_64/stpncpy.S | 28 +++ sysdeps/x86_64/strcat.S | 15 +- sysdeps/x86_64/strcpy.S | 12 +- sysdeps/x86_64/strncat.S | 28 +++ sysdeps/x86_64/strncpy.S | 27 +++ 30 files changed, 382 insertions(+), 158 deletions(-) create mode 100644 sysdeps/x86_64/stpncpy.S create mode 100644 sysdeps/x86_64/strncat.S create mode 100644 sysdeps/x86_64/strncpy.S