diff mbox series

[v1,4/4] x86: Add optimized functions for the wide-character strcpy family

Message ID 20221103085314.1069528-4-goldstein.w.n@gmail.com
State New
Headers show
Series [v1,1/4] benchtests: Make str{n}{cat|cpy} benchmarks output json | expand

Commit Message

Noah Goldstein Nov. 3, 2022, 8:53 a.m. UTC
Implemented:
    wcscat-avx2{+rtm}
    wcscpy-avx2{+rtm}
    wcpcpy-avx2{+rtm}
    wcsncpy-avx2{+rtm}
    wcpncpy-avx2{+rtm}
    wcsncat-avx2{+rtm}
    wcscat-evex
    wcscpy-evex
    wcpcpy-evex
    wcsncpy-evex
    wcpncpy-evex
    wcsncat-evex

Performance Changes:
    Times are from N = 10 runs of the benchmark suite and are reported
    as geometric mean of all ratios of New Implementation / Best Old
    Implementation. Best Old Implementation was determined with the
    highest ISA implementation.

    wcscat-avx2     -> 0.975
    wcscpy-avx2     -> 0.591
    wcpcpy-avx2     -> 0.698
    wcsncpy-avx2    -> 0.730
    wcpncpy-avx2    -> 0.711
    wcsncat-avx2    -> 0.938
    wcscat-evex     -> 0.991
    wcscpy-evex     -> 0.587
    wcpcpy-evex     -> 0.695
    wcsncpy-evex    -> 0.719
    wcpncpy-evex    -> 0.694
    wcsncat-evex    -> 0.959

Code Size Changes:
    This change (compared with the last two commits without it)
    increase the size of libc.so by 19392 bytes. For reference this
    entire patchset increases libc.so by 2624 bytes (so without the
    wide-character functions libc.so would decrease by 16768 bytes).

Full check passes on x86-64 and build succeeds for all ISA levels w/
and w/o multiarch.
---
Results attached to the evex/avx2 patches.
    
 sysdeps/x86_64/Makefile                     |   5 +
 sysdeps/x86_64/multiarch/Makefile           |  26 +++-
 sysdeps/x86_64/multiarch/ifunc-impl-list.c  | 135 +++++++++++++++++++-
 sysdeps/x86_64/multiarch/ifunc-wcs.h        |  60 +++++++++
 sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S  |   3 +
 sysdeps/x86_64/multiarch/wcpcpy-avx2.S      |   8 ++
 sysdeps/x86_64/multiarch/wcpcpy-evex.S      |   8 ++
 sysdeps/x86_64/multiarch/wcpcpy-generic.c   |  27 ++++
 sysdeps/x86_64/multiarch/wcpcpy.c           |  37 ++++++
 sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S |   3 +
 sysdeps/x86_64/multiarch/wcpncpy-avx2.S     |   8 ++
 sysdeps/x86_64/multiarch/wcpncpy-evex.S     |   8 ++
 sysdeps/x86_64/multiarch/wcpncpy-generic.c  |  27 ++++
 sysdeps/x86_64/multiarch/wcpncpy.c          |  37 ++++++
 sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S  |   3 +
 sysdeps/x86_64/multiarch/wcscat-avx2.S      |  10 ++
 sysdeps/x86_64/multiarch/wcscat-evex.S      |   9 ++
 sysdeps/x86_64/multiarch/wcscat-generic.c   |  27 ++++
 sysdeps/x86_64/multiarch/wcscat.c           |  37 ++++++
 sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S  |   3 +
 sysdeps/x86_64/multiarch/wcscpy-avx2.S      |   7 +
 sysdeps/x86_64/multiarch/wcscpy-evex.S      |   7 +
 sysdeps/x86_64/multiarch/wcscpy-generic.c   |   3 +-
 sysdeps/x86_64/multiarch/wcscpy.c           |  21 +++
 sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S |   3 +
 sysdeps/x86_64/multiarch/wcsncat-avx2.S     |   9 ++
 sysdeps/x86_64/multiarch/wcsncat-evex.S     |   9 ++
 sysdeps/x86_64/multiarch/wcsncat-generic.c  |  27 ++++
 sysdeps/x86_64/multiarch/wcsncat.c          |  34 +++++
 sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S |   3 +
 sysdeps/x86_64/multiarch/wcsncpy-avx2.S     |   7 +
 sysdeps/x86_64/multiarch/wcsncpy-evex.S     |   7 +
 sysdeps/x86_64/multiarch/wcsncpy-generic.c  |  27 ++++
 sysdeps/x86_64/multiarch/wcsncpy.c          |  37 ++++++
 sysdeps/x86_64/wcpcpy-generic.c             |  31 +++++
 sysdeps/x86_64/wcpcpy.S                     |  41 ++++++
 sysdeps/x86_64/wcpncpy-generic.c            |  31 +++++
 sysdeps/x86_64/wcpncpy.S                    |  41 ++++++
 sysdeps/x86_64/wcscat-generic.c             |  31 +++++
 sysdeps/x86_64/wcscat.S                     |  41 ++++++
 sysdeps/x86_64/wcscpy.S                     |   2 +
 sysdeps/x86_64/wcsncat-generic.c            |  31 +++++
 sysdeps/x86_64/wcsncat.S                    |  39 ++++++
 sysdeps/x86_64/wcsncpy-generic.c            |  31 +++++
 sysdeps/x86_64/wcsncpy.S                    |  41 ++++++
 45 files changed, 1036 insertions(+), 6 deletions(-)
 create mode 100644 sysdeps/x86_64/multiarch/ifunc-wcs.h
 create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-avx2.S
 create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-evex.S
 create mode 100644 sysdeps/x86_64/multiarch/wcpcpy-generic.c
 create mode 100644 sysdeps/x86_64/multiarch/wcpcpy.c
 create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-avx2.S
 create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-evex.S
 create mode 100644 sysdeps/x86_64/multiarch/wcpncpy-generic.c
 create mode 100644 sysdeps/x86_64/multiarch/wcpncpy.c
 create mode 100644 sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/wcscat-avx2.S
 create mode 100644 sysdeps/x86_64/multiarch/wcscat-evex.S
 create mode 100644 sysdeps/x86_64/multiarch/wcscat-generic.c
 create mode 100644 sysdeps/x86_64/multiarch/wcscat.c
 create mode 100644 sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/wcscpy-avx2.S
 create mode 100644 sysdeps/x86_64/multiarch/wcscpy-evex.S
 create mode 100644 sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/wcsncat-avx2.S
 create mode 100644 sysdeps/x86_64/multiarch/wcsncat-evex.S
 create mode 100644 sysdeps/x86_64/multiarch/wcsncat-generic.c
 create mode 100644 sysdeps/x86_64/multiarch/wcsncat.c
 create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S
 create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-avx2.S
 create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-evex.S
 create mode 100644 sysdeps/x86_64/multiarch/wcsncpy-generic.c
 create mode 100644 sysdeps/x86_64/multiarch/wcsncpy.c
 create mode 100644 sysdeps/x86_64/wcpcpy-generic.c
 create mode 100644 sysdeps/x86_64/wcpcpy.S
 create mode 100644 sysdeps/x86_64/wcpncpy-generic.c
 create mode 100644 sysdeps/x86_64/wcpncpy.S
 create mode 100644 sysdeps/x86_64/wcscat-generic.c
 create mode 100644 sysdeps/x86_64/wcscat.S
 create mode 100644 sysdeps/x86_64/wcsncat-generic.c
 create mode 100644 sysdeps/x86_64/wcsncat.S
 create mode 100644 sysdeps/x86_64/wcsncpy-generic.c
 create mode 100644 sysdeps/x86_64/wcsncpy.S
diff mbox series

Patch

diff --git a/sysdeps/x86_64/Makefile b/sysdeps/x86_64/Makefile
index 3627c5659f..688eb2d7c4 100644
--- a/sysdeps/x86_64/Makefile
+++ b/sysdeps/x86_64/Makefile
@@ -188,8 +188,13 @@  endif
 ifeq ($(subdir),wcsmbs)
 
 sysdep_routines += \
+  wcpcpy-generic \
+  wcpncpy-generic \
+  wcscat-generic \
   wcscpy-generic \
+  wcsncat-generic \
   wcsncmp-generic \
+  wcsncpy-generic \
   wcsnlen-generic \
 # sysdep_routines
 
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile
index 597ac9d5e9..8b5a3a4ee5 100644
--- a/sysdeps/x86_64/multiarch/Makefile
+++ b/sysdeps/x86_64/multiarch/Makefile
@@ -130,6 +130,18 @@  endif
 
 ifeq ($(subdir),wcsmbs)
 sysdep_routines += \
+  wcpcpy-avx2 \
+  wcpcpy-avx2-rtm \
+  wcpcpy-evex \
+  wcpcpy-generic \
+  wcpncpy-avx2 \
+  wcpncpy-avx2-rtm \
+  wcpncpy-evex \
+  wcpncpy-generic \
+  wcscat-avx2 \
+  wcscat-avx2-rtm \
+  wcscat-evex \
+  wcscat-generic \
   wcschr-avx2 \
   wcschr-avx2-rtm \
   wcschr-evex \
@@ -139,6 +151,10 @@  sysdep_routines += \
   wcscmp-avx2-rtm \
   wcscmp-evex \
   wcscmp-sse2 \
+  wcscpy-avx2 \
+  wcscpy-avx2-rtm \
+  wcscpy-evex \
+  wcscpy-generic \
   wcscpy-ssse3 \
   wcslen-avx2 \
   wcslen-avx2-rtm \
@@ -146,9 +162,17 @@  sysdep_routines += \
   wcslen-evex512 \
   wcslen-sse2 \
   wcslen-sse4_1 \
+  wcsncat-avx2 \
+  wcsncat-avx2-rtm \
+  wcsncat-evex \
+  wcsncat-generic \
   wcsncmp-avx2 \
   wcsncmp-avx2-rtm \
   wcsncmp-evex \
+  wcsncpy-avx2 \
+  wcsncpy-avx2-rtm \
+  wcsncpy-evex \
+  wcsncpy-generic \
   wcsnlen-avx2 \
   wcsnlen-avx2-rtm \
   wcsnlen-evex \
@@ -161,8 +185,8 @@  sysdep_routines += \
   wmemchr-avx2 \
   wmemchr-avx2-rtm \
   wmemchr-evex \
-  wmemchr-evex512 \
   wmemchr-evex-rtm \
+  wmemchr-evex512 \
   wmemchr-sse2 \
   wmemcmp-avx2-movbe \
   wmemcmp-avx2-movbe-rtm \
diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
index c3d75a09f4..f31fdf4d05 100644
--- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c
@@ -891,16 +891,145 @@  __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/x86_64/multiarch/wcscpy.c.  */
   IFUNC_IMPL (i, name, wcscpy,
-	      /* ISA V4 wrapper for SSSE3 implementation because
-	         the SSSE3 implementation is also used at ISA
-	         level 3/4.  */
 	      X86_IFUNC_IMPL_ADD_V4 (array, i, wcscpy,
+				     (CPU_FEATURE_USABLE (AVX512VL)
+				      && CPU_FEATURE_USABLE (AVX512BW)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcscpy_evex)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcscpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcscpy_avx2)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcscpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)
+				      && CPU_FEATURE_USABLE (RTM)),
+				     __wcscpy_avx2_rtm)
+	      X86_IFUNC_IMPL_ADD_V2 (array, i, wcscpy,
 				     CPU_FEATURE_USABLE (SSSE3),
 				     __wcscpy_ssse3)
 	      X86_IFUNC_IMPL_ADD_V1 (array, i, wcscpy,
 				     1,
 				     __wcscpy_generic))
 
+  /* Support sysdeps/x86_64/multiarch/wcsncpy.c.  */
+  IFUNC_IMPL (i, name, wcsncpy,
+	      X86_IFUNC_IMPL_ADD_V4 (array, i, wcsncpy,
+				     (CPU_FEATURE_USABLE (AVX512VL)
+				      && CPU_FEATURE_USABLE (AVX512BW)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcsncpy_evex)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcsncpy_avx2)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)
+				      && CPU_FEATURE_USABLE (RTM)),
+				     __wcsncpy_avx2_rtm)
+	      X86_IFUNC_IMPL_ADD_V1 (array, i, wcpncpy,
+				     1,
+				     __wcsncpy_generic))
+
+  /* Support sysdeps/x86_64/multiarch/wcpcpy.c.  */
+  IFUNC_IMPL (i, name, wcpcpy,
+	      X86_IFUNC_IMPL_ADD_V4 (array, i, wcpcpy,
+				     (CPU_FEATURE_USABLE (AVX512VL)
+				      && CPU_FEATURE_USABLE (AVX512BW)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcpcpy_evex)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcpcpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcpcpy_avx2)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcpcpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)
+				      && CPU_FEATURE_USABLE (RTM)),
+				     __wcpcpy_avx2_rtm)
+	      X86_IFUNC_IMPL_ADD_V1 (array, i, wcpcpy,
+				     1,
+				     __wcpcpy_generic))
+
+  /* Support sysdeps/x86_64/multiarch/wcpncpy.c.  */
+  IFUNC_IMPL (i, name, wcpncpy,
+	      X86_IFUNC_IMPL_ADD_V4 (array, i, wcpncpy,
+				     (CPU_FEATURE_USABLE (AVX512VL)
+				      && CPU_FEATURE_USABLE (AVX512BW)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcpncpy_evex)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcpncpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcpncpy_avx2)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcpncpy,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)
+				      && CPU_FEATURE_USABLE (RTM)),
+				     __wcpncpy_avx2_rtm)
+	      X86_IFUNC_IMPL_ADD_V1 (array, i, wcsncpy,
+				     1,
+				     __wcpncpy_generic))
+
+  /* Support sysdeps/x86_64/multiarch/wcscat.c.  */
+  IFUNC_IMPL (i, name, wcscat,
+	      X86_IFUNC_IMPL_ADD_V4 (array, i, wcscat,
+				     (CPU_FEATURE_USABLE (AVX512VL)
+				      && CPU_FEATURE_USABLE (AVX512BW)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcscat_evex)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcscat,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcscat_avx2)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcscat,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)
+				      && CPU_FEATURE_USABLE (RTM)),
+				     __wcscat_avx2_rtm)
+	      X86_IFUNC_IMPL_ADD_V1 (array, i, wcscat,
+				     1,
+				     __wcscat_generic))
+
+  /* Support sysdeps/x86_64/multiarch/wcsncat.c.  */
+  IFUNC_IMPL (i, name, wcsncat,
+	      X86_IFUNC_IMPL_ADD_V4 (array, i, wcsncat,
+				     (CPU_FEATURE_USABLE (AVX512VL)
+				      && CPU_FEATURE_USABLE (AVX512BW)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcsncat_evex)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncat,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)),
+				     __wcsncat_avx2)
+	      X86_IFUNC_IMPL_ADD_V3 (array, i, wcsncat,
+				     (CPU_FEATURE_USABLE (AVX2)
+				      && CPU_FEATURE_USABLE (BMI1)
+				      && CPU_FEATURE_USABLE (BMI2)
+				      && CPU_FEATURE_USABLE (RTM)),
+				     __wcsncat_avx2_rtm)
+	      X86_IFUNC_IMPL_ADD_V1 (array, i, wcsncat,
+				     1,
+				     __wcsncat_generic))
+
   /* Support sysdeps/x86_64/multiarch/wcslen.c.  */
   IFUNC_IMPL (i, name, wcslen,
 	      X86_IFUNC_IMPL_ADD_V4 (array, i, wcslen,
diff --git a/sysdeps/x86_64/multiarch/ifunc-wcs.h b/sysdeps/x86_64/multiarch/ifunc-wcs.h
new file mode 100644
index 0000000000..cda633d8fb
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/ifunc-wcs.h
@@ -0,0 +1,60 @@ 
+/* Common definition for ifunc selections optimized wide-character
+   string copy functions.
+
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <init-arch.h>
+
+#ifndef GENERIC
+# define GENERIC generic
+#endif
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (GENERIC) attribute_hidden;
+
+static inline void *
+IFUNC_SELECTOR (void)
+{
+  const struct cpu_features *cpu_features = __get_cpu_features ();
+
+  if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+      && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI1)
+      && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+      && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+				      AVX_Fast_Unaligned_Load, ))
+    {
+      if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+	  && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+	return OPTIMIZE (evex);
+
+      if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+	return OPTIMIZE (avx2_rtm);
+
+      if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features,
+				       Prefer_No_VZEROUPPER, !))
+	return OPTIMIZE (avx2);
+
+    }
+
+  return OPTIMIZE (GENERIC);
+}
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S
new file mode 100644
index 0000000000..756280a3ab
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-avx2-rtm.S
@@ -0,0 +1,3 @@ 
+#define WCPCPY	__wcpcpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcpcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-avx2.S b/sysdeps/x86_64/multiarch/wcpcpy-avx2.S
new file mode 100644
index 0000000000..0fffd912d3
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-avx2.S
@@ -0,0 +1,8 @@ 
+#ifndef WCPCPY
+# define WCPCPY	__wcpcpy_avx2
+#endif
+
+#define USE_AS_STPCPY
+#define USE_AS_WCSCPY
+#define STRCPY	WCPCPY
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-evex.S b/sysdeps/x86_64/multiarch/wcpcpy-evex.S
new file mode 100644
index 0000000000..ac6429cc07
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-evex.S
@@ -0,0 +1,8 @@ 
+#ifndef WCPCPY
+# define WCPCPY	__wcpcpy_evex
+#endif
+
+#define USE_AS_STPCPY
+#define USE_AS_WCSCPY
+#define STRCPY	WCPCPY
+#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcpcpy-generic.c b/sysdeps/x86_64/multiarch/wcpcpy-generic.c
new file mode 100644
index 0000000000..0ba29b081f
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy-generic.c
@@ -0,0 +1,27 @@ 
+/* wcpcpy.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+   be able to fallback to it.  */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCPCPY __wcpcpy_generic
+# include <wcsmbs/wcpcpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcpcpy.c b/sysdeps/x86_64/multiarch/wcpcpy.c
new file mode 100644
index 0000000000..8f96ddbc99
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpcpy.c
@@ -0,0 +1,37 @@ 
+/* Multiple versions of wcpcpy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.  */
+#if IS_IN (libc)
+# define __wcpcpy __redirect_wcpcpy
+# include <wchar.h>
+# undef __wcpcpy
+
+# define SYMBOL_NAME wcpcpy
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcpcpy, __wcpcpy, IFUNC_SELECTOR ());
+weak_alias (__wcpcpy, wcpcpy)
+# ifdef SHARED
+__hidden_ver1 (__wcpcpy, __GI___wcpcpy, __redirect_wcpcpy)
+  __attribute__((visibility ("hidden"))) __attribute_copy__ (wcpcpy);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S
new file mode 100644
index 0000000000..80600d6b01
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-avx2-rtm.S
@@ -0,0 +1,3 @@ 
+#define WCPNCPY	__wcpncpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcpncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-avx2.S b/sysdeps/x86_64/multiarch/wcpncpy-avx2.S
new file mode 100644
index 0000000000..b7e594f7b7
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-avx2.S
@@ -0,0 +1,8 @@ 
+#ifndef WCPNCPY
+# define WCPNCPY	__wcpncpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STPCPY
+#define STRNCPY	WCPNCPY
+#include "strncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-evex.S b/sysdeps/x86_64/multiarch/wcpncpy-evex.S
new file mode 100644
index 0000000000..62ddb694fe
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-evex.S
@@ -0,0 +1,8 @@ 
+#ifndef WCPNCPY
+# define WCPNCPY	__wcpncpy_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STPCPY
+#define STRNCPY	WCPNCPY
+#include "strncpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcpncpy-generic.c b/sysdeps/x86_64/multiarch/wcpncpy-generic.c
new file mode 100644
index 0000000000..4aab4ecdd2
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy-generic.c
@@ -0,0 +1,27 @@ 
+/* wcpncpy.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+   be able to fallback to it.  */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCPNCPY __wcpncpy_generic
+# include <wcsmbs/wcpncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcpncpy.c b/sysdeps/x86_64/multiarch/wcpncpy.c
new file mode 100644
index 0000000000..ed8f307e07
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcpncpy.c
@@ -0,0 +1,37 @@ 
+/* Multiple versions of wcpncpy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.  */
+#if IS_IN (libc)
+# define __wcpncpy __redirect_wcpncpy
+# include <wchar.h>
+# undef __wcpncpy
+
+# define SYMBOL_NAME wcpncpy
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcpncpy, __wcpncpy, IFUNC_SELECTOR ());
+weak_alias (__wcpncpy, wcpncpy)
+# ifdef SHARED
+__hidden_ver1 (__wcpncpy, __GI___wcpncpy, __redirect_wcpncpy)
+  __attribute__((visibility ("hidden"))) __attribute_copy__ (wcpncpy);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S
new file mode 100644
index 0000000000..e99449a2dc
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-avx2-rtm.S
@@ -0,0 +1,3 @@ 
+#define WCSCAT	__wcscat_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcscat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscat-avx2.S b/sysdeps/x86_64/multiarch/wcscat-avx2.S
new file mode 100644
index 0000000000..a20f23c09d
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-avx2.S
@@ -0,0 +1,10 @@ 
+#ifndef WCSCAT
+# define WCSCAT	__wcscat_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRCPY	WCSCAT
+
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscat-evex.S b/sysdeps/x86_64/multiarch/wcscat-evex.S
new file mode 100644
index 0000000000..1d017e4899
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-evex.S
@@ -0,0 +1,9 @@ 
+#ifndef WCSCAT
+# define WCSCAT	__wcscat_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRCPY	WCSCAT
+#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcscat-generic.c b/sysdeps/x86_64/multiarch/wcscat-generic.c
new file mode 100644
index 0000000000..6476f85bbb
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat-generic.c
@@ -0,0 +1,27 @@ 
+/* wcscat.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+   be able to fallback to it.  */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCSCAT __wcscat_generic
+# include <wcsmbs/wcscat.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcscat.c b/sysdeps/x86_64/multiarch/wcscat.c
new file mode 100644
index 0000000000..3277c44561
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscat.c
@@ -0,0 +1,37 @@ 
+/* Multiple versions of wcscat.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.  */
+#if IS_IN (libc)
+# define __wcscat __redirect_wcscat
+# include <wchar.h>
+# undef __wcscat
+
+# define SYMBOL_NAME wcscat
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcscat, __wcscat, IFUNC_SELECTOR ());
+weak_alias (__wcscat, wcscat)
+# ifdef SHARED
+__hidden_ver1 (__wcscat, __GI___wcscat, __redirect_wcscat)
+  __attribute__((visibility ("hidden"))) __attribute_copy__ (wcscat);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S
new file mode 100644
index 0000000000..2f800c8d3e
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscpy-avx2-rtm.S
@@ -0,0 +1,3 @@ 
+#define WCSCPY	__wcscpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcscpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscpy-avx2.S b/sysdeps/x86_64/multiarch/wcscpy-avx2.S
new file mode 100644
index 0000000000..6bc509da07
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscpy-avx2.S
@@ -0,0 +1,7 @@ 
+#ifndef WCSCPY
+# define WCSCPY	__wcscpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define STRCPY	WCSCPY
+#include "strcpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcscpy-evex.S b/sysdeps/x86_64/multiarch/wcscpy-evex.S
new file mode 100644
index 0000000000..1069a8e224
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcscpy-evex.S
@@ -0,0 +1,7 @@ 
+#ifndef WCSCPY
+# define WCSCPY	__wcscpy_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define STRCPY	WCSCPY
+#include "strcpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcscpy-generic.c b/sysdeps/x86_64/multiarch/wcscpy-generic.c
index 93d314aaad..600d606c45 100644
--- a/sysdeps/x86_64/multiarch/wcscpy-generic.c
+++ b/sysdeps/x86_64/multiarch/wcscpy-generic.c
@@ -18,8 +18,7 @@ 
 
 
 #include <isa-level.h>
-
-#if ISA_SHOULD_BUILD (1)
+#if ISA_SHOULD_BUILD (2)
 
 # define WCSCPY  __wcscpy_generic
 # include <wcsmbs/wcscpy.c>
diff --git a/sysdeps/x86_64/multiarch/wcscpy.c b/sysdeps/x86_64/multiarch/wcscpy.c
index 92c917b6b4..7f6387817b 100644
--- a/sysdeps/x86_64/multiarch/wcscpy.c
+++ b/sysdeps/x86_64/multiarch/wcscpy.c
@@ -26,6 +26,11 @@ 
 # define SYMBOL_NAME wcscpy
 # include <init-arch.h>
 
+extern __typeof (REDIRECT_NAME) OPTIMIZE (evex) attribute_hidden;
+
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2) attribute_hidden;
+extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_rtm) attribute_hidden;
+
 extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden;
 
 extern __typeof (REDIRECT_NAME) OPTIMIZE (generic) attribute_hidden;
@@ -35,6 +40,22 @@  IFUNC_SELECTOR (void)
 {
   const struct cpu_features* cpu_features = __get_cpu_features ();
 
+  if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX2)
+      && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI1)
+      && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, BMI2)
+      && X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load, ))
+    {
+      if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512VL)
+	  && X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, AVX512BW))
+	return OPTIMIZE (evex);
+
+      if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
+	return OPTIMIZE (avx2_rtm);
+
+      if (X86_ISA_CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER, !))
+	return OPTIMIZE (avx2);
+    }
+
   if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features, SSSE3))
     return OPTIMIZE (ssse3);
 
diff --git a/sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S
new file mode 100644
index 0000000000..609d6e69c0
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-avx2-rtm.S
@@ -0,0 +1,3 @@ 
+#define WCSNCAT	__wcsncat_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcsncat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncat-avx2.S b/sysdeps/x86_64/multiarch/wcsncat-avx2.S
new file mode 100644
index 0000000000..a72105b7e9
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-avx2.S
@@ -0,0 +1,9 @@ 
+#ifndef WCSNCAT
+# define WCSNCAT	__wcsncat_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRNCAT	WCSNCAT
+#include "strncat-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncat-evex.S b/sysdeps/x86_64/multiarch/wcsncat-evex.S
new file mode 100644
index 0000000000..392215950a
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-evex.S
@@ -0,0 +1,9 @@ 
+#ifndef WCSCAT
+# define WCSCAT	__wcsncat_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define USE_AS_STRCAT
+
+#define STRNCAT	WCSCAT
+#include "strncat-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncat-generic.c b/sysdeps/x86_64/multiarch/wcsncat-generic.c
new file mode 100644
index 0000000000..9ced02b35e
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat-generic.c
@@ -0,0 +1,27 @@ 
+/* wcsncat.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+   be able to fallback to it.  */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCSNCAT __wcsncat_generic
+# include <wcsmbs/wcsncat.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcsncat.c b/sysdeps/x86_64/multiarch/wcsncat.c
new file mode 100644
index 0000000000..49c46aef08
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncat.c
@@ -0,0 +1,34 @@ 
+/* Multiple versions of wcsncat.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.  */
+#if IS_IN (libc)
+# define wcsncat __redirect_wcsncat
+# include <wchar.h>
+# undef wcsncat
+
+# define SYMBOL_NAME wcsncat
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcsncat, wcsncat, IFUNC_SELECTOR ());
+# ifdef SHARED
+__hidden_ver1 (wcsncat, __GI_wcsncat, __redirect_wcsncat)
+  __attribute__((visibility ("hidden"))) __attribute_copy__ (wcsncat);
+# endif
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S b/sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S
new file mode 100644
index 0000000000..cab5a6b820
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-avx2-rtm.S
@@ -0,0 +1,3 @@ 
+#define WCSNCPY	__wcsncpy_avx2_rtm
+#include "x86-avx2-rtm-vecs.h"
+#include "wcsncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-avx2.S b/sysdeps/x86_64/multiarch/wcsncpy-avx2.S
new file mode 100644
index 0000000000..3a1a8a372c
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-avx2.S
@@ -0,0 +1,7 @@ 
+#ifndef WCSNCPY
+# define WCSNCPY	__wcsncpy_avx2
+#endif
+
+#define USE_AS_WCSCPY
+#define STRNCPY	WCSNCPY
+#include "strncpy-avx2.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-evex.S b/sysdeps/x86_64/multiarch/wcsncpy-evex.S
new file mode 100644
index 0000000000..2debb8fd6b
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-evex.S
@@ -0,0 +1,7 @@ 
+#ifndef WCSNCPY
+# define WCSNCPY	__wcsncpy_evex
+#endif
+
+#define USE_AS_WCSCPY
+#define STRNCPY	WCSNCPY
+#include "strncpy-evex.S"
diff --git a/sysdeps/x86_64/multiarch/wcsncpy-generic.c b/sysdeps/x86_64/multiarch/wcsncpy-generic.c
new file mode 100644
index 0000000000..693521713b
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy-generic.c
@@ -0,0 +1,27 @@ 
+/* wcsncpy.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* We always need to build this implementation as strspn-sse4 needs to
+   be able to fallback to it.  */
+#include <isa-level.h>
+#if ISA_SHOULD_BUILD (2)
+
+# define WCSNCPY __wcsncpy_generic
+# include <wcsmbs/wcsncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/multiarch/wcsncpy.c b/sysdeps/x86_64/multiarch/wcsncpy.c
new file mode 100644
index 0000000000..5b89dd4d27
--- /dev/null
+++ b/sysdeps/x86_64/multiarch/wcsncpy.c
@@ -0,0 +1,37 @@ 
+/* Multiple versions of wcsncpy.
+   All versions must be listed in ifunc-impl-list.c.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* Define multiple versions only for the definition in libc.  */
+#if IS_IN (libc)
+# define __wcsncpy __redirect_wcsncpy
+# include <wchar.h>
+# undef __wcsncpy
+
+# define SYMBOL_NAME wcsncpy
+# include <init-arch.h>
+
+# include "ifunc-wcs.h"
+
+libc_ifunc_redirected (__redirect_wcsncpy, __wcsncpy, IFUNC_SELECTOR ());
+weak_alias (__wcsncpy, wcsncpy)
+# ifdef SHARED
+__hidden_ver1 (__wcsncpy, __GI___wcsncpy, __redirect_wcsncpy)
+  __attribute__((visibility ("hidden"))) __attribute_copy__ (wcsncpy);
+# endif
+#endif
diff --git a/sysdeps/x86_64/wcpcpy-generic.c b/sysdeps/x86_64/wcpcpy-generic.c
new file mode 100644
index 0000000000..d52525f288
--- /dev/null
+++ b/sysdeps/x86_64/wcpcpy-generic.c
@@ -0,0 +1,31 @@ 
+/* wcpcpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcpcpy non-multiarch build is split into two files,
+   wcpcpy-generic.c and wcpcpy.S. The wcpcpy-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcpcpy-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcpcpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcpcpy.S b/sysdeps/x86_64/wcpcpy.S
new file mode 100644
index 0000000000..ec32dc070a
--- /dev/null
+++ b/sysdeps/x86_64/wcpcpy.S
@@ -0,0 +1,41 @@ 
+/* wcpcpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcpcpy non-multiarch build is split into two files,
+   wcpcpy-generic.c and wcpcpy.S. The wcpcpy-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcpcpy-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCPCPY	__wcpcpy
+
+# define DEFAULT_IMPL_V4	"multiarch/wcpcpy-evex.S"
+# define DEFAULT_IMPL_V3	"multiarch/wcpcpy-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+   should never be used from here.  */
+# define DEFAULT_IMPL_V1	"ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcpcpy, wcpcpy)
+libc_hidden_def (__wcpcpy)
+#endif
diff --git a/sysdeps/x86_64/wcpncpy-generic.c b/sysdeps/x86_64/wcpncpy-generic.c
new file mode 100644
index 0000000000..871219a445
--- /dev/null
+++ b/sysdeps/x86_64/wcpncpy-generic.c
@@ -0,0 +1,31 @@ 
+/* wcpncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcpncpy non-multiarch build is split into two files,
+   wcpncpy-generic.c and wcpncpy.S. The wcpncpy-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcpncpy-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcpncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcpncpy.S b/sysdeps/x86_64/wcpncpy.S
new file mode 100644
index 0000000000..68e6ff1836
--- /dev/null
+++ b/sysdeps/x86_64/wcpncpy.S
@@ -0,0 +1,41 @@ 
+/* wcpncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcpncpy non-multiarch build is split into two files,
+   wcpncpy-generic.c and wcpncpy.S. The wcpncpy-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcpncpy-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCPNCPY	__wcpncpy
+
+# define DEFAULT_IMPL_V4	"multiarch/wcpncpy-evex.S"
+# define DEFAULT_IMPL_V3	"multiarch/wcpncpy-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+   should never be used from here.  */
+# define DEFAULT_IMPL_V1	"ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcpncpy, wcpncpy)
+libc_hidden_def (__wcpncpy)
+#endif
diff --git a/sysdeps/x86_64/wcscat-generic.c b/sysdeps/x86_64/wcscat-generic.c
new file mode 100644
index 0000000000..85f981a81f
--- /dev/null
+++ b/sysdeps/x86_64/wcscat-generic.c
@@ -0,0 +1,31 @@ 
+/* wcscat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcscat non-multiarch build is split into two files,
+   wcscat-generic.c and wcscat.S. The wcscat-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcscat-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcscat.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcscat.S b/sysdeps/x86_64/wcscat.S
new file mode 100644
index 0000000000..007de3c40c
--- /dev/null
+++ b/sysdeps/x86_64/wcscat.S
@@ -0,0 +1,41 @@ 
+/* wcscat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcscat non-multiarch build is split into two files,
+   wcscat-generic.c and wcscat.S. The wcscat-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcscat-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCSCAT	__wcscat
+
+# define DEFAULT_IMPL_V4	"multiarch/wcscat-evex.S"
+# define DEFAULT_IMPL_V3	"multiarch/wcscat-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+   should never be used from here.  */
+# define DEFAULT_IMPL_V1	"ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcscat, wcscat)
+libc_hidden_def (__wcscat)
+#endif
diff --git a/sysdeps/x86_64/wcscpy.S b/sysdeps/x86_64/wcscpy.S
index 11d0bb4bab..ab9288ed74 100644
--- a/sysdeps/x86_64/wcscpy.S
+++ b/sysdeps/x86_64/wcscpy.S
@@ -28,6 +28,8 @@ 
 
 # define WCSCPY	__wcscpy
 
+# define DEFAULT_IMPL_V4	"multiarch/wcscpy-evex.S"
+# define DEFAULT_IMPL_V3	"multiarch/wcscpy-avx2.S"
 # define DEFAULT_IMPL_V2	"multiarch/wcscpy-ssse3.S"
 /* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
    should never be used from here.  */
diff --git a/sysdeps/x86_64/wcsncat-generic.c b/sysdeps/x86_64/wcsncat-generic.c
new file mode 100644
index 0000000000..2cc0f7b11a
--- /dev/null
+++ b/sysdeps/x86_64/wcsncat-generic.c
@@ -0,0 +1,31 @@ 
+/* wcsncat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcsncat non-multiarch build is split into two files,
+   wcsncat-generic.c and wcsncat.S. The wcsncat-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcsncat-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcsncat.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcsncat.S b/sysdeps/x86_64/wcsncat.S
new file mode 100644
index 0000000000..3f4c7948db
--- /dev/null
+++ b/sysdeps/x86_64/wcsncat.S
@@ -0,0 +1,39 @@ 
+/* wcsncat dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcsncat non-multiarch build is split into two files,
+   wcsncat-generic.c and wcsncat.S. The wcsncat-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcsncat-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCSNCAT	wcsncat
+
+# define DEFAULT_IMPL_V4	"multiarch/wcsncat-evex.S"
+# define DEFAULT_IMPL_V3	"multiarch/wcsncat-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+   should never be used from here.  */
+# define DEFAULT_IMPL_V1	"ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+#endif
diff --git a/sysdeps/x86_64/wcsncpy-generic.c b/sysdeps/x86_64/wcsncpy-generic.c
new file mode 100644
index 0000000000..49d06b8ae8
--- /dev/null
+++ b/sysdeps/x86_64/wcsncpy-generic.c
@@ -0,0 +1,31 @@ 
+/* wcsncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcsncpy non-multiarch build is split into two files,
+   wcsncpy-generic.c and wcsncpy.S. The wcsncpy-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcsncpy-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL <= 3
+
+# include <wcsmbs/wcsncpy.c>
+
+#endif
diff --git a/sysdeps/x86_64/wcsncpy.S b/sysdeps/x86_64/wcsncpy.S
new file mode 100644
index 0000000000..e1428fd4c1
--- /dev/null
+++ b/sysdeps/x86_64/wcsncpy.S
@@ -0,0 +1,41 @@ 
+/* wcsncpy dispatch for RTLD and non-multiarch .c ISA level 1 build.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+/* wcsncpy non-multiarch build is split into two files,
+   wcsncpy-generic.c and wcsncpy.S. The wcsncpy-generic.c build is
+   for ISA level <= 1 and just uses multiarch/wcsncpy-generic.c.
+   This must be split into two files because we cannot include C
+   code from assembly or vice versa.  */
+
+#include <isa-level.h>
+
+#if MINIMUM_X86_ISA_LEVEL >= 4
+
+# define WCSNCPY	__wcsncpy
+
+# define DEFAULT_IMPL_V4	"multiarch/wcsncpy-evex.S"
+# define DEFAULT_IMPL_V3	"multiarch/wcsncpy-avx2.S"
+/* isa-default-impl.h expects DEFAULT_IMPL_V1 to be defined but it
+   should never be used from here.  */
+# define DEFAULT_IMPL_V1	"ERROR -- Invalid ISA IMPL"
+
+# include "isa-default-impl.h"
+
+weak_alias (__wcsncpy, wcsncpy)
+libc_hidden_def (__wcsncpy)
+#endif