| Message ID | 20211027024323.1199441-3-goldstein.w.n@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | [v1,1/6] String: Add __memcmpeq as build target | expand |
On Tue, Oct 26, 2021 at 7:43 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > No bug. This commit adds support for __memcmpeq to be implemented > seperately from memcmp. Support is added for versions optimized with > sse2, avx2, and evex. > --- > sysdeps/generic/ifunc-init.h | 5 +- > sysdeps/x86_64/memcmp.S | 9 ++-- > sysdeps/x86_64/multiarch/Makefile | 4 ++ > sysdeps/x86_64/multiarch/ifunc-impl-list.c | 21 +++++++++ > sysdeps/x86_64/multiarch/ifunc-memcmpeq.h | 49 ++++++++++++++++++++ > sysdeps/x86_64/multiarch/memcmp-sse2.S | 4 +- > sysdeps/x86_64/multiarch/memcmp.c | 3 -- > sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S | 12 +++++ > sysdeps/x86_64/multiarch/memcmpeq-avx2.S | 23 +++++++++ > sysdeps/x86_64/multiarch/memcmpeq-evex.S | 23 +++++++++ > sysdeps/x86_64/multiarch/memcmpeq-sse2.S | 23 +++++++++ > sysdeps/x86_64/multiarch/memcmpeq.c | 35 ++++++++++++++ > 12 files changed, 202 insertions(+), 9 deletions(-) > create mode 100644 sysdeps/x86_64/multiarch/ifunc-memcmpeq.h > create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S > create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-avx2.S > create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-evex.S > create mode 100644 sysdeps/x86_64/multiarch/memcmpeq-sse2.S > create mode 100644 sysdeps/x86_64/multiarch/memcmpeq.c > > diff --git a/sysdeps/generic/ifunc-init.h b/sysdeps/generic/ifunc-init.h > index 7f69485de8..ee8a8289c8 100644 > --- a/sysdeps/generic/ifunc-init.h > +++ b/sysdeps/generic/ifunc-init.h > @@ -50,5 +50,8 @@ > '__<symbol>_<variant>' as the optimized implementation and > '<symbol>_ifunc_selector' as the IFUNC selector. */ > #define REDIRECT_NAME EVALUATOR1 (__redirect, SYMBOL_NAME) > -#define OPTIMIZE(name) EVALUATOR2 (SYMBOL_NAME, name) > #define IFUNC_SELECTOR EVALUATOR1 (SYMBOL_NAME, ifunc_selector) > +#define OPTIMIZE1(name) EVALUATOR1 (SYMBOL_NAME, name) > +#define OPTIMIZE2(name) EVALUATOR2 (SYMBOL_NAME, name) > +/* Default is to use OPTIMIZE2. */ > +#define OPTIMIZE(name) OPTIMIZE2(name) > diff --git a/sysdeps/x86_64/memcmp.S b/sysdeps/x86_64/memcmp.S > index 8a03e572e8..b53f2c0866 100644 > --- a/sysdeps/x86_64/memcmp.S > +++ b/sysdeps/x86_64/memcmp.S > @@ -356,9 +356,10 @@ L(ATR32res): > .p2align 4,, 4 > END(memcmp) > > -#undef bcmp > +#ifdef USE_AS_MEMCMPEQ > +libc_hidden_def (memcmp) > +#else > +# undef bcmp > weak_alias (memcmp, bcmp) > -#undef __memcmpeq > -strong_alias (memcmp, __memcmpeq) > libc_hidden_builtin_def (memcmp) > -libc_hidden_def (__memcmpeq) > +#endif > diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile > index 26be40959c..044778585b 100644 > --- a/sysdeps/x86_64/multiarch/Makefile > +++ b/sysdeps/x86_64/multiarch/Makefile > @@ -7,7 +7,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ > memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \ > memrchr-sse2 memrchr-avx2 \ > memcmp-sse2 \ > + memcmpeq-sse2 \ > memcmp-avx2-movbe \ > + memcmpeq-avx2 \ > memcmp-sse4 memcpy-ssse3 \ > memmove-ssse3 \ > memcpy-ssse3-back \ > @@ -42,6 +44,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ > memset-avx512-unaligned-erms \ > memchr-avx2-rtm \ > memcmp-avx2-movbe-rtm \ > + memcmpeq-avx2-rtm \ > memmove-avx-unaligned-erms-rtm \ > memrchr-avx2-rtm \ > memset-avx2-unaligned-erms-rtm \ > @@ -61,6 +64,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ > strrchr-avx2-rtm \ > memchr-evex \ > memcmp-evex-movbe \ > + memcmpeq-evex \ > memmove-evex-unaligned-erms \ > memrchr-evex \ > memset-evex-unaligned-erms \ > diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c > index 39ab10613b..f7f3806d1d 100644 > --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c > +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c > @@ -38,6 +38,27 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, > > size_t i = 0; > > + /* Support sysdeps/x86_64/multiarch/memcmpeq.c. */ > + IFUNC_IMPL (i, name, __memcmpeq, > + IFUNC_IMPL_ADD (array, i, __memcmpeq, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (MOVBE) > + && CPU_FEATURE_USABLE (BMI2)), > + __memcmpeq_avx2) > + IFUNC_IMPL_ADD (array, i, __memcmpeq, > + (CPU_FEATURE_USABLE (AVX2) > + && CPU_FEATURE_USABLE (BMI2) > + && CPU_FEATURE_USABLE (MOVBE) > + && CPU_FEATURE_USABLE (RTM)), > + __memcmpeq_avx2_rtm) > + IFUNC_IMPL_ADD (array, i, __memcmpeq, > + (CPU_FEATURE_USABLE (AVX512VL) > + && CPU_FEATURE_USABLE (AVX512BW) > + && CPU_FEATURE_USABLE (MOVBE) > + && CPU_FEATURE_USABLE (BMI2)), > + __memcmpeq_evex) > + IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmpeq_sse2)) > + > /* Support sysdeps/x86_64/multiarch/memchr.c. */ > IFUNC_IMPL (i, name, memchr, > IFUNC_IMPL_ADD (array, i, memchr, > diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h b/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h > new file mode 100644 > index 0000000000..3319a9568a > --- /dev/null > +++ b/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h > @@ -0,0 +1,49 @@ > +/* Common definition for __memcmpeq ifunc selections. > + All versions must be listed in ifunc-impl-list.c. > + Copyright (C) 2017-2021 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +# include <init-arch.h> > + > +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (sse2) attribute_hidden; > +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2) attribute_hidden; > +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2_rtm) attribute_hidden; > +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (evex) attribute_hidden; > + > +static inline void * > +IFUNC_SELECTOR (void) > +{ > + const struct cpu_features* cpu_features = __get_cpu_features (); > + > + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) > + && CPU_FEATURE_USABLE_P (cpu_features, BMI2) > + && CPU_FEATURE_USABLE_P (cpu_features, MOVBE) > + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) > + { > + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) > + && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) > + return OPTIMIZE1 (evex); > + > + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) > + return OPTIMIZE1 (avx2_rtm); > + > + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) > + return OPTIMIZE1 (avx2); > + } > + > + return OPTIMIZE1 (sse2); > +} > diff --git a/sysdeps/x86_64/multiarch/memcmp-sse2.S b/sysdeps/x86_64/multiarch/memcmp-sse2.S > index 7b30b7ca2e..132d6fb339 100644 > --- a/sysdeps/x86_64/multiarch/memcmp-sse2.S > +++ b/sysdeps/x86_64/multiarch/memcmp-sse2.S > @@ -17,7 +17,9 @@ > <https://www.gnu.org/licenses/>. */ > > #if IS_IN (libc) > -# define memcmp __memcmp_sse2 > +# ifndef memcmp > +# define memcmp __memcmp_sse2 > +# endif > > # ifdef SHARED > # undef libc_hidden_builtin_def > diff --git a/sysdeps/x86_64/multiarch/memcmp.c b/sysdeps/x86_64/multiarch/memcmp.c > index 7b3409b1dd..fe725f3563 100644 > --- a/sysdeps/x86_64/multiarch/memcmp.c > +++ b/sysdeps/x86_64/multiarch/memcmp.c > @@ -29,9 +29,6 @@ > libc_ifunc_redirected (__redirect_memcmp, memcmp, IFUNC_SELECTOR ()); > # undef bcmp > weak_alias (memcmp, bcmp) > -# undef __memcmpeq > -strong_alias (memcmp, __memcmpeq) > -libc_hidden_def (__memcmpeq) > > # ifdef SHARED > __hidden_ver1 (memcmp, __GI_memcmp, __redirect_memcmp) > diff --git a/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S b/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S > new file mode 100644 > index 0000000000..24b6a0c9ff > --- /dev/null > +++ b/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S > @@ -0,0 +1,12 @@ > +#ifndef MEMCMP > +# define MEMCMP __memcmpeq_avx2_rtm > +#endif > + > +#define ZERO_UPPER_VEC_REGISTERS_RETURN \ > + ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST > + > +#define VZEROUPPER_RETURN jmp L(return_vzeroupper) > + > +#define SECTION(p) p##.avx.rtm > + > +#include "memcmpeq-avx2.S" > diff --git a/sysdeps/x86_64/multiarch/memcmpeq-avx2.S b/sysdeps/x86_64/multiarch/memcmpeq-avx2.S > new file mode 100644 > index 0000000000..0181ea0d8d > --- /dev/null > +++ b/sysdeps/x86_64/multiarch/memcmpeq-avx2.S > @@ -0,0 +1,23 @@ > +/* __memcmpeq optimized with AVX2. > + Copyright (C) 2017-2021 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#ifndef MEMCMP > +# define MEMCMP __memcmpeq_avx2 > +#endif > + > +#include "memcmp-avx2-movbe.S" > diff --git a/sysdeps/x86_64/multiarch/memcmpeq-evex.S b/sysdeps/x86_64/multiarch/memcmpeq-evex.S > new file mode 100644 > index 0000000000..951e1e9560 > --- /dev/null > +++ b/sysdeps/x86_64/multiarch/memcmpeq-evex.S > @@ -0,0 +1,23 @@ > +/* __memcmpeq optimized with EVEX. > + Copyright (C) 2017-2021 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#ifndef MEMCMP > +# define MEMCMP __memcmpeq_evex > +#endif > + > +#include "memcmp-evex-movbe.S" > diff --git a/sysdeps/x86_64/multiarch/memcmpeq-sse2.S b/sysdeps/x86_64/multiarch/memcmpeq-sse2.S > new file mode 100644 > index 0000000000..c488cbbcd9 > --- /dev/null > +++ b/sysdeps/x86_64/multiarch/memcmpeq-sse2.S > @@ -0,0 +1,23 @@ > +/* __memcmpeq optimized with SSE2. > + Copyright (C) 2017-2021 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#ifndef memcmp > +# define memcmp __memcmpeq_sse2 > +#endif > +#define USE_AS_MEMCMPEQ 1 > +#include "memcmp-sse2.S" > diff --git a/sysdeps/x86_64/multiarch/memcmpeq.c b/sysdeps/x86_64/multiarch/memcmpeq.c > new file mode 100644 > index 0000000000..163e56047e > --- /dev/null > +++ b/sysdeps/x86_64/multiarch/memcmpeq.c > @@ -0,0 +1,35 @@ > +/* Multiple versions of __memcmpeq. > + All versions must be listed in ifunc-impl-list.c. > + Copyright (C) 2017-2021 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +/* Define multiple versions only for the definition in libc. */ > +#if IS_IN (libc) > +# define __memcmpeq __redirect___memcmpeq > +# include <string.h> > +# undef __memcmpeq > + > +# define SYMBOL_NAME __memcmpeq > +# include "ifunc-memcmpeq.h" > + > +libc_ifunc_redirected (__redirect___memcmpeq, __memcmpeq, IFUNC_SELECTOR ()); > + > +# ifdef SHARED > +__hidden_ver1 (__memcmpeq, __GI___memcmpeq, __redirect___memcmpeq) > + __attribute__ ((visibility ("hidden"))) __attribute_copy__ (__memcmpeq); > +# endif > +#endif > -- > 2.25.1 > LGTM. Reviewed-by: H.J. Lu <hjl.tools@gmail.com> Thanks.
diff --git a/sysdeps/generic/ifunc-init.h b/sysdeps/generic/ifunc-init.h index 7f69485de8..ee8a8289c8 100644 --- a/sysdeps/generic/ifunc-init.h +++ b/sysdeps/generic/ifunc-init.h @@ -50,5 +50,8 @@ '__<symbol>_<variant>' as the optimized implementation and '<symbol>_ifunc_selector' as the IFUNC selector. */ #define REDIRECT_NAME EVALUATOR1 (__redirect, SYMBOL_NAME) -#define OPTIMIZE(name) EVALUATOR2 (SYMBOL_NAME, name) #define IFUNC_SELECTOR EVALUATOR1 (SYMBOL_NAME, ifunc_selector) +#define OPTIMIZE1(name) EVALUATOR1 (SYMBOL_NAME, name) +#define OPTIMIZE2(name) EVALUATOR2 (SYMBOL_NAME, name) +/* Default is to use OPTIMIZE2. */ +#define OPTIMIZE(name) OPTIMIZE2(name) diff --git a/sysdeps/x86_64/memcmp.S b/sysdeps/x86_64/memcmp.S index 8a03e572e8..b53f2c0866 100644 --- a/sysdeps/x86_64/memcmp.S +++ b/sysdeps/x86_64/memcmp.S @@ -356,9 +356,10 @@ L(ATR32res): .p2align 4,, 4 END(memcmp) -#undef bcmp +#ifdef USE_AS_MEMCMPEQ +libc_hidden_def (memcmp) +#else +# undef bcmp weak_alias (memcmp, bcmp) -#undef __memcmpeq -strong_alias (memcmp, __memcmpeq) libc_hidden_builtin_def (memcmp) -libc_hidden_def (__memcmpeq) +#endif diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 26be40959c..044778585b 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -7,7 +7,9 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \ memrchr-sse2 memrchr-avx2 \ memcmp-sse2 \ + memcmpeq-sse2 \ memcmp-avx2-movbe \ + memcmpeq-avx2 \ memcmp-sse4 memcpy-ssse3 \ memmove-ssse3 \ memcpy-ssse3-back \ @@ -42,6 +44,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ memset-avx512-unaligned-erms \ memchr-avx2-rtm \ memcmp-avx2-movbe-rtm \ + memcmpeq-avx2-rtm \ memmove-avx-unaligned-erms-rtm \ memrchr-avx2-rtm \ memset-avx2-unaligned-erms-rtm \ @@ -61,6 +64,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c \ strrchr-avx2-rtm \ memchr-evex \ memcmp-evex-movbe \ + memcmpeq-evex \ memmove-evex-unaligned-erms \ memrchr-evex \ memset-evex-unaligned-erms \ diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c index 39ab10613b..f7f3806d1d 100644 --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c @@ -38,6 +38,27 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, size_t i = 0; + /* Support sysdeps/x86_64/multiarch/memcmpeq.c. */ + IFUNC_IMPL (i, name, __memcmpeq, + IFUNC_IMPL_ADD (array, i, __memcmpeq, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (MOVBE) + && CPU_FEATURE_USABLE (BMI2)), + __memcmpeq_avx2) + IFUNC_IMPL_ADD (array, i, __memcmpeq, + (CPU_FEATURE_USABLE (AVX2) + && CPU_FEATURE_USABLE (BMI2) + && CPU_FEATURE_USABLE (MOVBE) + && CPU_FEATURE_USABLE (RTM)), + __memcmpeq_avx2_rtm) + IFUNC_IMPL_ADD (array, i, __memcmpeq, + (CPU_FEATURE_USABLE (AVX512VL) + && CPU_FEATURE_USABLE (AVX512BW) + && CPU_FEATURE_USABLE (MOVBE) + && CPU_FEATURE_USABLE (BMI2)), + __memcmpeq_evex) + IFUNC_IMPL_ADD (array, i, __memcmpeq, 1, __memcmpeq_sse2)) + /* Support sysdeps/x86_64/multiarch/memchr.c. */ IFUNC_IMPL (i, name, memchr, IFUNC_IMPL_ADD (array, i, memchr, diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h b/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h new file mode 100644 index 0000000000..3319a9568a --- /dev/null +++ b/sysdeps/x86_64/multiarch/ifunc-memcmpeq.h @@ -0,0 +1,49 @@ +/* Common definition for __memcmpeq ifunc selections. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017-2021 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +# include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (sse2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (avx2_rtm) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE1 (evex) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (CPU_FEATURE_USABLE_P (cpu_features, AVX2) + && CPU_FEATURE_USABLE_P (cpu_features, BMI2) + && CPU_FEATURE_USABLE_P (cpu_features, MOVBE) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + { + if (CPU_FEATURE_USABLE_P (cpu_features, AVX512VL) + && CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)) + return OPTIMIZE1 (evex); + + if (CPU_FEATURE_USABLE_P (cpu_features, RTM)) + return OPTIMIZE1 (avx2_rtm); + + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER)) + return OPTIMIZE1 (avx2); + } + + return OPTIMIZE1 (sse2); +} diff --git a/sysdeps/x86_64/multiarch/memcmp-sse2.S b/sysdeps/x86_64/multiarch/memcmp-sse2.S index 7b30b7ca2e..132d6fb339 100644 --- a/sysdeps/x86_64/multiarch/memcmp-sse2.S +++ b/sysdeps/x86_64/multiarch/memcmp-sse2.S @@ -17,7 +17,9 @@ <https://www.gnu.org/licenses/>. */ #if IS_IN (libc) -# define memcmp __memcmp_sse2 +# ifndef memcmp +# define memcmp __memcmp_sse2 +# endif # ifdef SHARED # undef libc_hidden_builtin_def diff --git a/sysdeps/x86_64/multiarch/memcmp.c b/sysdeps/x86_64/multiarch/memcmp.c index 7b3409b1dd..fe725f3563 100644 --- a/sysdeps/x86_64/multiarch/memcmp.c +++ b/sysdeps/x86_64/multiarch/memcmp.c @@ -29,9 +29,6 @@ libc_ifunc_redirected (__redirect_memcmp, memcmp, IFUNC_SELECTOR ()); # undef bcmp weak_alias (memcmp, bcmp) -# undef __memcmpeq -strong_alias (memcmp, __memcmpeq) -libc_hidden_def (__memcmpeq) # ifdef SHARED __hidden_ver1 (memcmp, __GI_memcmp, __redirect_memcmp) diff --git a/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S b/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S new file mode 100644 index 0000000000..24b6a0c9ff --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcmpeq-avx2-rtm.S @@ -0,0 +1,12 @@ +#ifndef MEMCMP +# define MEMCMP __memcmpeq_avx2_rtm +#endif + +#define ZERO_UPPER_VEC_REGISTERS_RETURN \ + ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST + +#define VZEROUPPER_RETURN jmp L(return_vzeroupper) + +#define SECTION(p) p##.avx.rtm + +#include "memcmpeq-avx2.S" diff --git a/sysdeps/x86_64/multiarch/memcmpeq-avx2.S b/sysdeps/x86_64/multiarch/memcmpeq-avx2.S new file mode 100644 index 0000000000..0181ea0d8d --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcmpeq-avx2.S @@ -0,0 +1,23 @@ +/* __memcmpeq optimized with AVX2. + Copyright (C) 2017-2021 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#ifndef MEMCMP +# define MEMCMP __memcmpeq_avx2 +#endif + +#include "memcmp-avx2-movbe.S" diff --git a/sysdeps/x86_64/multiarch/memcmpeq-evex.S b/sysdeps/x86_64/multiarch/memcmpeq-evex.S new file mode 100644 index 0000000000..951e1e9560 --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcmpeq-evex.S @@ -0,0 +1,23 @@ +/* __memcmpeq optimized with EVEX. + Copyright (C) 2017-2021 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#ifndef MEMCMP +# define MEMCMP __memcmpeq_evex +#endif + +#include "memcmp-evex-movbe.S" diff --git a/sysdeps/x86_64/multiarch/memcmpeq-sse2.S b/sysdeps/x86_64/multiarch/memcmpeq-sse2.S new file mode 100644 index 0000000000..c488cbbcd9 --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcmpeq-sse2.S @@ -0,0 +1,23 @@ +/* __memcmpeq optimized with SSE2. + Copyright (C) 2017-2021 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#ifndef memcmp +# define memcmp __memcmpeq_sse2 +#endif +#define USE_AS_MEMCMPEQ 1 +#include "memcmp-sse2.S" diff --git a/sysdeps/x86_64/multiarch/memcmpeq.c b/sysdeps/x86_64/multiarch/memcmpeq.c new file mode 100644 index 0000000000..163e56047e --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcmpeq.c @@ -0,0 +1,35 @@ +/* Multiple versions of __memcmpeq. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017-2021 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in libc. */ +#if IS_IN (libc) +# define __memcmpeq __redirect___memcmpeq +# include <string.h> +# undef __memcmpeq + +# define SYMBOL_NAME __memcmpeq +# include "ifunc-memcmpeq.h" + +libc_ifunc_redirected (__redirect___memcmpeq, __memcmpeq, IFUNC_SELECTOR ()); + +# ifdef SHARED +__hidden_ver1 (__memcmpeq, __GI___memcmpeq, __redirect___memcmpeq) + __attribute__ ((visibility ("hidden"))) __attribute_copy__ (__memcmpeq); +# endif +#endif