[v9,4/6] x86: Update memset to use new VEC macros

Replace %VEC(n) -> %VMM(n)

This commit does not change libc.so

Tested build on x86-64
---
 .../memset-avx2-unaligned-erms-rtm.S          |  8 +--
 .../multiarch/memset-avx2-unaligned-erms.S    | 14 +---
 .../multiarch/memset-avx512-unaligned-erms.S  | 20 +-----
 .../multiarch/memset-evex-unaligned-erms.S    | 20 +-----
 .../multiarch/memset-sse2-unaligned-erms.S    | 10 +--
 .../multiarch/memset-vec-unaligned-erms.S     | 70 ++++++++-----------
 6 files changed, 43 insertions(+), 99 deletions(-)

On Fri, Oct 14, 2022 at 5:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote:
>
> Replace %VEC(n) -> %VMM(n)
>
> This commit does not change libc.so
>
> Tested build on x86-64
> ---
>  .../memset-avx2-unaligned-erms-rtm.S          |  8 +--
>  .../multiarch/memset-avx2-unaligned-erms.S    | 14 +---
>  .../multiarch/memset-avx512-unaligned-erms.S  | 20 +-----
>  .../multiarch/memset-evex-unaligned-erms.S    | 20 +-----
>  .../multiarch/memset-sse2-unaligned-erms.S    | 10 +--
>  .../multiarch/memset-vec-unaligned-erms.S     | 70 ++++++++-----------
>  6 files changed, 43 insertions(+), 99 deletions(-)
>
> diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
> index 8ac3e479bb..bc8605faf3 100644
> --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
> +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms-rtm.S
> @@ -1,10 +1,6 @@
> -#define ZERO_UPPER_VEC_REGISTERS_RETURN \
> -  ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST
> +#include "x86-avx-rtm-vecs.h"
>
> -#define VZEROUPPER_RETURN jmp   L(return)
> -
> -#define SECTION(p) p##.avx.rtm
>  #define MEMSET_SYMBOL(p,s)     p##_avx2_##s##_rtm
>  #define WMEMSET_SYMBOL(p,s)    p##_avx2_##s##_rtm
>
> -#include "memset-avx2-unaligned-erms.S"
> +# include "memset-avx2-unaligned-erms.S"
> diff --git a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
> index a9054a9122..47cf5072a4 100644
> --- a/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-avx2-unaligned-erms.S
> @@ -4,14 +4,9 @@
>
>  # define USE_WITH_AVX2 1
>
> -# define VEC_SIZE      32
> -# define MOV_SIZE      4
> -# define RET_SIZE      4
> -
> -# define VEC(i)                ymm##i
> -
> -# define VMOVU     vmovdqu
> -# define VMOVA     vmovdqa
> +# ifndef VEC_SIZE
> +#  include "x86-avx-vecs.h"
> +# endif
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
>    vmovd d, %xmm0; \
> @@ -26,9 +21,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH() vpbroadcastd %xmm0, %ymm0
>  # define WMEMSET_VDUP_TO_VEC0_LOW() vpbroadcastd %xmm0, %xmm0
>
> -# ifndef SECTION
> -#  define SECTION(p)           p##.avx
> -# endif
>  # ifndef MEMSET_SYMBOL
>  #  define MEMSET_SYMBOL(p,s)   p##_avx2_##s
>  # endif
> diff --git a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
> index 47623b8ee8..84145b6c27 100644
> --- a/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-avx512-unaligned-erms.S
> @@ -4,26 +4,14 @@
>
>  # define USE_WITH_AVX512       1
>
> -# define VEC_SIZE      64
> -# define MOV_SIZE      6
> -# define RET_SIZE      1
> -
> -# define XMM0          xmm16
> -# define YMM0          ymm16
> -# define VEC0          zmm16
> -# define VEC(i)                VEC##i
> -
> -# define VMOVU     vmovdqu64
> -# define VMOVA     vmovdqa64
> -
> -# define VZEROUPPER
> +# include "x86-evex512-vecs.h"
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastb d, %VEC0; \
> +  vpbroadcastb d, %VMM(0); \
>    movq r, %rax
>
>  # define WMEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastd d, %VEC0; \
> +  vpbroadcastd d, %VMM(0); \
>    movq r, %rax
>
>  # define MEMSET_VDUP_TO_VEC0_HIGH()
> @@ -32,8 +20,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH()
>  # define WMEMSET_VDUP_TO_VEC0_LOW()
>
> -# define SECTION(p)            p##.evex512
> -
>  #ifndef MEMSET_SYMBOL
>  # define MEMSET_SYMBOL(p,s)    p##_avx512_##s
>  #endif
> diff --git a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
> index ac4b2d2d50..1f03b26bf8 100644
> --- a/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-evex-unaligned-erms.S
> @@ -4,26 +4,14 @@
>
>  # define USE_WITH_EVEX 1
>
> -# define VEC_SIZE      32
> -# define MOV_SIZE      6
> -# define RET_SIZE      1
> -
> -# define XMM0          xmm16
> -# define YMM0          ymm16
> -# define VEC0          ymm16
> -# define VEC(i)                VEC##i
> -
> -# define VMOVU     vmovdqu64
> -# define VMOVA     vmovdqa64
> -
> -# define VZEROUPPER
> +# include "x86-evex256-vecs.h"
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastb d, %VEC0; \
> +  vpbroadcastb d, %VMM(0); \
>    movq r, %rax
>
>  # define WMEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
> -  vpbroadcastd d, %VEC0; \
> +  vpbroadcastd d, %VMM(0); \
>    movq r, %rax
>
>  # define MEMSET_VDUP_TO_VEC0_HIGH()
> @@ -32,8 +20,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH()
>  # define WMEMSET_VDUP_TO_VEC0_LOW()
>
> -# define SECTION(p)            p##.evex
> -
>  #ifndef MEMSET_SYMBOL
>  # define MEMSET_SYMBOL(p,s)    p##_evex_##s
>  #endif
> diff --git a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
> index 44f9b8888b..34b245d8ca 100644
> --- a/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-sse2-unaligned-erms.S
> @@ -26,13 +26,7 @@
>  # include <sysdep.h>
>  # define USE_WITH_SSE2 1
>
> -# define VEC_SIZE      16
> -# define MOV_SIZE      3
> -# define RET_SIZE      1
> -
> -# define VEC(i)                xmm##i
> -# define VMOVU     movups
> -# define VMOVA     movaps
> +# include "x86-sse2-vecs.h"
>
>  # define MEMSET_SET_VEC0_AND_SET_RETURN(d, r) \
>    movd d, %xmm0; \
> @@ -52,8 +46,6 @@
>  # define WMEMSET_VDUP_TO_VEC0_HIGH()
>  # define WMEMSET_VDUP_TO_VEC0_LOW()
>
> -# define SECTION(p)            p
> -
>  # ifndef MEMSET_SYMBOL
>  #  define MEMSET_SYMBOL(p,s)   p##_sse2_##s
>  # endif
> diff --git a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
> index 905d0fa464..03de0ab907 100644
> --- a/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
> +++ b/sysdeps/x86_64/multiarch/memset-vec-unaligned-erms.S
> @@ -34,14 +34,6 @@
>  # define WMEMSET_CHK_SYMBOL(p,s)       WMEMSET_SYMBOL(p, s)
>  #endif
>
> -#ifndef XMM0
> -# define XMM0                          xmm0
> -#endif
> -
> -#ifndef YMM0
> -# define YMM0                          ymm0
> -#endif
> -
>  #ifndef VZEROUPPER
>  # if VEC_SIZE > 16
>  #  define VZEROUPPER                   vzeroupper
> @@ -150,8 +142,8 @@ L(entry_from_wmemset):
>         cmpq    $(VEC_SIZE * 2), %rdx
>         ja      L(more_2x_vec)
>         /* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
> -       VMOVU   %VEC(0), -VEC_SIZE(%rdi,%rdx)
> -       VMOVU   %VEC(0), (%rdi)
> +       VMOVU   %VMM(0), -VEC_SIZE(%rdi,%rdx)
> +       VMOVU   %VMM(0), (%rdi)
>         VZEROUPPER_RETURN
>  #if defined USE_MULTIARCH && IS_IN (libc)
>  END (MEMSET_SYMBOL (__memset, unaligned))
> @@ -175,19 +167,19 @@ ENTRY_P2ALIGN (MEMSET_SYMBOL (__memset, unaligned_erms), 6)
>         cmp     $(VEC_SIZE * 2), %RDX_LP
>         ja      L(stosb_more_2x_vec)
>         /* From VEC and to 2 * VEC.  No branch when size == VEC_SIZE.  */
> -       VMOVU   %VEC(0), (%rdi)
> -       VMOVU   %VEC(0), (VEC_SIZE * -1)(%rdi, %rdx)
> +       VMOVU   %VMM(0), (%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
>         VZEROUPPER_RETURN
>  #endif
>
>         .p2align 4,, 4
>  L(last_2x_vec):
>  #ifdef USE_LESS_VEC_MASK_STORE
> -       VMOVU   %VEC(0), (VEC_SIZE * -2)(%rdi, %rdx)
> -       VMOVU   %VEC(0), (VEC_SIZE * -1)(%rdi, %rdx)
> +       VMOVU   %VMM(0), (VEC_SIZE * -2)(%rdi, %rdx)
> +       VMOVU   %VMM(0), (VEC_SIZE * -1)(%rdi, %rdx)
>  #else
> -       VMOVU   %VEC(0), (VEC_SIZE * -2)(%rdi)
> -       VMOVU   %VEC(0), (VEC_SIZE * -1)(%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * -2)(%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * -1)(%rdi)
>  #endif
>         VZEROUPPER_RETURN
>
> @@ -221,7 +213,7 @@ L(less_vec_from_wmemset):
>         bzhil   %edx, %ecx, %ecx
>         kmovd   %ecx, %k1
>  # endif
> -       vmovdqu8 %VEC(0), (%rax){%k1}
> +       vmovdqu8 %VMM(0), (%rax){%k1}
>         VZEROUPPER_RETURN
>
>  # if defined USE_MULTIARCH && IS_IN (libc)
> @@ -249,8 +241,8 @@ L(stosb_more_2x_vec):
>            and (4x, 8x] jump to target.  */
>  L(more_2x_vec):
>         /* Store next 2x vec regardless.  */
> -       VMOVU   %VEC(0), (%rdi)
> -       VMOVU   %VEC(0), (VEC_SIZE * 1)(%rdi)
> +       VMOVU   %VMM(0), (%rdi)
> +       VMOVU   %VMM(0), (VEC_SIZE * 1)(%rdi)
>
>
>         /* Two different methods of setting up pointers / compare. The two
> @@ -278,8 +270,8 @@ L(more_2x_vec):
>  #endif
>
>         /* Store next 2x vec regardless.  */
> -       VMOVU   %VEC(0), (VEC_SIZE * 2)(%rax)
> -       VMOVU   %VEC(0), (VEC_SIZE * 3)(%rax)
> +       VMOVU   %VMM(0), (VEC_SIZE * 2)(%rax)
> +       VMOVU   %VMM(0), (VEC_SIZE * 3)(%rax)
>
>
>  #if defined USE_WITH_EVEX || defined USE_WITH_AVX512
> @@ -304,20 +296,20 @@ L(more_2x_vec):
>         andq    $(VEC_SIZE * -2), %LOOP_REG
>         .p2align 4
>  L(loop):
> -       VMOVA   %VEC(0), LOOP_4X_OFFSET(%LOOP_REG)
> -       VMOVA   %VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
> -       VMOVA   %VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
> -       VMOVA   %VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
> +       VMOVA   %VMM(0), LOOP_4X_OFFSET(%LOOP_REG)
> +       VMOVA   %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%LOOP_REG)
> +       VMOVA   %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%LOOP_REG)
> +       VMOVA   %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%LOOP_REG)
>         subq    $-(VEC_SIZE * 4), %LOOP_REG
>         cmpq    %END_REG, %LOOP_REG
>         jb      L(loop)
>         .p2align 4,, MOV_SIZE
>  L(last_4x_vec):
> -       VMOVU   %VEC(0), LOOP_4X_OFFSET(%END_REG)
> -       VMOVU   %VEC(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
> -       VMOVU   %VEC(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
> -       VMOVU   %VEC(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
> -L(return):
> +       VMOVU   %VMM(0), LOOP_4X_OFFSET(%END_REG)
> +       VMOVU   %VMM(0), (VEC_SIZE + LOOP_4X_OFFSET)(%END_REG)
> +       VMOVU   %VMM(0), (VEC_SIZE * 2 + LOOP_4X_OFFSET)(%END_REG)
> +       VMOVU   %VMM(0), (VEC_SIZE * 3 + LOOP_4X_OFFSET)(%END_REG)
> +L(return_vzeroupper):
>  #if VEC_SIZE > 16
>         ZERO_UPPER_VEC_REGISTERS_RETURN
>  #else
> @@ -355,7 +347,7 @@ L(cross_page):
>         jge     L(between_16_31)
>  #endif
>  #ifndef USE_XMM_LESS_VEC
> -       MOVQ    %XMM0, %SET_REG64
> +       MOVQ    %VMM_128(0), %SET_REG64
>  #endif
>         cmpl    $8, %edx
>         jge     L(between_8_15)
> @@ -374,8 +366,8 @@ L(between_0_0):
>         .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, RET_SIZE)
>         /* From 32 to 63.  No branch when size == 32.  */
>  L(between_32_63):
> -       VMOVU   %YMM0, (%LESS_VEC_REG)
> -       VMOVU   %YMM0, -32(%LESS_VEC_REG, %rdx)
> +       VMOVU   %VMM_256(0), (%LESS_VEC_REG)
> +       VMOVU   %VMM_256(0), -32(%LESS_VEC_REG, %rdx)
>         VZEROUPPER_RETURN
>  #endif
>
> @@ -383,8 +375,8 @@ L(between_32_63):
>         .p2align 4,, SMALL_MEMSET_ALIGN(MOV_SIZE, 1)
>  L(between_16_31):
>         /* From 16 to 31.  No branch when size == 16.  */
> -       VMOVU   %XMM0, (%LESS_VEC_REG)
> -       VMOVU   %XMM0, -16(%LESS_VEC_REG, %rdx)
> +       VMOVU   %VMM_128(0), (%LESS_VEC_REG)
> +       VMOVU   %VMM_128(0), -16(%LESS_VEC_REG, %rdx)
>         ret
>  #endif
>
> @@ -394,8 +386,8 @@ L(between_16_31):
>  L(between_8_15):
>         /* From 8 to 15.  No branch when size == 8.  */
>  #ifdef USE_XMM_LESS_VEC
> -       MOVQ    %XMM0, (%rdi)
> -       MOVQ    %XMM0, -8(%rdi, %rdx)
> +       MOVQ    %VMM_128(0), (%rdi)
> +       MOVQ    %VMM_128(0), -8(%rdi, %rdx)
>  #else
>         movq    %SET_REG64, (%LESS_VEC_REG)
>         movq    %SET_REG64, -8(%LESS_VEC_REG, %rdx)
> @@ -408,8 +400,8 @@ L(between_8_15):
>  L(between_4_7):
>         /* From 4 to 7.  No branch when size == 4.  */
>  #ifdef USE_XMM_LESS_VEC
> -       MOVD    %XMM0, (%rdi)
> -       MOVD    %XMM0, -4(%rdi, %rdx)
> +       MOVD    %VMM_128(0), (%rdi)
> +       MOVD    %VMM_128(0), -4(%rdi, %rdx)
>  #else
>         movl    %SET_REG32, (%LESS_VEC_REG)
>         movl    %SET_REG32, -4(%LESS_VEC_REG, %rdx)
> --
> 2.34.1
>

LGTM.

Thanks.