| Message ID | 20221015002100.129511-3-goldstein.w.n@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | [v9,1/6] x86: Update VEC macros to complete API for evex/evex512 impls | expand |
On Fri, Oct 14, 2022 at 5:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > Replace %VEC(n) -> %VMM(n) > > This commit does not change libc.so > > Tested build on x86-64 > --- > .../memmove-avx-unaligned-erms-rtm.S | 15 +- > .../multiarch/memmove-avx-unaligned-erms.S | 9 +- > .../multiarch/memmove-avx512-unaligned-erms.S | 30 +- > .../multiarch/memmove-evex-unaligned-erms.S | 30 +- > .../multiarch/memmove-sse2-unaligned-erms.S | 11 +- > .../multiarch/memmove-vec-unaligned-erms.S | 262 +++++++++--------- > 6 files changed, 135 insertions(+), 222 deletions(-) > > diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S > index 67a55f0c85..c2a95dc247 100644 > --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S > +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S > @@ -1,16 +1,9 @@ > -#if IS_IN (libc) Keep this. > -# define VEC_SIZE 32 > -# define VEC(i) ymm##i > -# define VMOVNT vmovntdq > -# define VMOVU vmovdqu > -# define VMOVA vmovdqa > -# define MOV_SIZE 4 > -# define ZERO_UPPER_VEC_REGISTERS_RETURN \ > - ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST > +#include <isa-level.h> > > -# define VZEROUPPER_RETURN jmp L(return) > +#if ISA_SHOULD_BUILD (3) This ISA_SHOULD_BUILD change isn't needed. > +# include "x86-avx-rtm-vecs.h" > > -# define SECTION(p) p##.avx.rtm > # define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_rtm > > # include "memmove-vec-unaligned-erms.S" > diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S > index a14b155667..4e4b4635f9 100644 > --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S > +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S > @@ -2,14 +2,7 @@ > > #if ISA_SHOULD_BUILD (3) > > -# define VEC_SIZE 32 > -# define VEC(i) ymm##i > -# define VMOVNT vmovntdq > -# define VMOVU vmovdqu > -# define VMOVA vmovdqa > -# define MOV_SIZE 4 > - > -# define SECTION(p) p##.avx > +# include "x86-avx-vecs.h" > > # ifndef MEMMOVE_SYMBOL > # define MEMMOVE_SYMBOL(p,s) p##_avx_##s > diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S > index 8d1568a7ba..cca97e38f8 100644 > --- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S > +++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S > @@ -2,35 +2,7 @@ > > #if ISA_SHOULD_BUILD (4) > > -# define VEC_SIZE 64 > -# define XMM0 xmm16 > -# define XMM1 xmm17 > -# define YMM0 ymm16 > -# define YMM1 ymm17 > -# define VEC0 zmm16 > -# define VEC1 zmm17 > -# define VEC2 zmm18 > -# define VEC3 zmm19 > -# define VEC4 zmm20 > -# define VEC5 zmm21 > -# define VEC6 zmm22 > -# define VEC7 zmm23 > -# define VEC8 zmm24 > -# define VEC9 zmm25 > -# define VEC10 zmm26 > -# define VEC11 zmm27 > -# define VEC12 zmm28 > -# define VEC13 zmm29 > -# define VEC14 zmm30 > -# define VEC15 zmm31 > -# define VEC(i) VEC##i > -# define VMOVNT vmovntdq > -# define VMOVU vmovdqu64 > -# define VMOVA vmovdqa64 > -# define VZEROUPPER > -# define MOV_SIZE 6 > - > -# define SECTION(p) p##.evex512 > +# include "x86-evex512-vecs.h" > > # ifndef MEMMOVE_SYMBOL > # define MEMMOVE_SYMBOL(p,s) p##_avx512_##s > diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S > index 2373017358..1f7b5715f7 100644 > --- a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S > +++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S > @@ -2,35 +2,7 @@ > > #if ISA_SHOULD_BUILD (4) > > -# define VEC_SIZE 32 > -# define XMM0 xmm16 > -# define XMM1 xmm17 > -# define YMM0 ymm16 > -# define YMM1 ymm17 > -# define VEC0 ymm16 > -# define VEC1 ymm17 > -# define VEC2 ymm18 > -# define VEC3 ymm19 > -# define VEC4 ymm20 > -# define VEC5 ymm21 > -# define VEC6 ymm22 > -# define VEC7 ymm23 > -# define VEC8 ymm24 > -# define VEC9 ymm25 > -# define VEC10 ymm26 > -# define VEC11 ymm27 > -# define VEC12 ymm28 > -# define VEC13 ymm29 > -# define VEC14 ymm30 > -# define VEC15 ymm31 > -# define VEC(i) VEC##i > -# define VMOVNT vmovntdq > -# define VMOVU vmovdqu64 > -# define VMOVA vmovdqa64 > -# define VZEROUPPER > -# define MOV_SIZE 6 > - > -# define SECTION(p) p##.evex > +# include "x86-evex256-vecs.h" > > # ifndef MEMMOVE_SYMBOL > # define MEMMOVE_SYMBOL(p,s) p##_evex_##s > diff --git a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S > index 422a079902..8431bcd000 100644 > --- a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S > +++ b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S > @@ -22,18 +22,9 @@ > so we need this to build for ISA V2 builds. */ > #if ISA_SHOULD_BUILD (2) > > -# include <sysdep.h> > +# include "x86-sse2-vecs.h" > > -# define VEC_SIZE 16 > -# define VEC(i) xmm##i > # define PREFETCHNT prefetchnta > -# define VMOVNT movntdq > -/* Use movups and movaps for smaller code sizes. */ > -# define VMOVU movups > -# define VMOVA movaps > -# define MOV_SIZE 3 > - > -# define SECTION(p) p > > # ifndef MEMMOVE_SYMBOL > # define MEMMOVE_SYMBOL(p,s) p##_sse2_##s > diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S > index 04747133b7..5b758cae5e 100644 > --- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S > +++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S > @@ -60,14 +60,6 @@ > # define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) > #endif > > -#ifndef XMM0 > -# define XMM0 xmm0 > -#endif > - > -#ifndef YMM0 > -# define YMM0 ymm0 > -#endif > - > #ifndef VZEROUPPER > # if VEC_SIZE > 16 > # define VZEROUPPER vzeroupper > @@ -225,13 +217,13 @@ L(start): > cmp $VEC_SIZE, %RDX_LP > jb L(less_vec) > /* Load regardless. */ > - VMOVU (%rsi), %VEC(0) > + VMOVU (%rsi), %VMM(0) > cmp $(VEC_SIZE * 2), %RDX_LP > ja L(more_2x_vec) > /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ > - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) > - VMOVU %VEC(0), (%rdi) > - VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) > + VMOVU -VEC_SIZE(%rsi,%rdx), %VMM(1) > + VMOVU %VMM(0), (%rdi) > + VMOVU %VMM(1), -VEC_SIZE(%rdi,%rdx) > #if !(defined USE_MULTIARCH && IS_IN (libc)) > ZERO_UPPER_VEC_REGISTERS_RETURN > #else > @@ -270,15 +262,15 @@ L(start_erms): > cmp $VEC_SIZE, %RDX_LP > jb L(less_vec) > /* Load regardless. */ > - VMOVU (%rsi), %VEC(0) > + VMOVU (%rsi), %VMM(0) > cmp $(VEC_SIZE * 2), %RDX_LP > ja L(movsb_more_2x_vec) > /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. > */ > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(1) > - VMOVU %VEC(0), (%rdi) > - VMOVU %VEC(1), -VEC_SIZE(%rdi, %rdx) > -L(return): > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(1) > + VMOVU %VMM(0), (%rdi) > + VMOVU %VMM(1), -VEC_SIZE(%rdi, %rdx) > +L(return_vzeroupper): > # if VEC_SIZE > 16 > ZERO_UPPER_VEC_REGISTERS_RETURN > # else > @@ -359,10 +351,10 @@ L(between_16_31): > .p2align 4,, 10 > L(between_32_63): > /* From 32 to 63. No branch when size == 32. */ > - VMOVU (%rsi), %YMM0 > - VMOVU -32(%rsi, %rdx), %YMM1 > - VMOVU %YMM0, (%rdi) > - VMOVU %YMM1, -32(%rdi, %rdx) > + VMOVU (%rsi), %VMM_256(0) > + VMOVU -32(%rsi, %rdx), %VMM_256(1) > + VMOVU %VMM_256(0), (%rdi) > + VMOVU %VMM_256(1), -32(%rdi, %rdx) > VZEROUPPER_RETURN > #endif > > @@ -380,12 +372,12 @@ L(last_4x_vec): > /* Copy from 2 * VEC + 1 to 4 * VEC, inclusively. */ > > /* VEC(0) and VEC(1) have already been loaded. */ > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(2) > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(3) > - VMOVU %VEC(0), (%rdi) > - VMOVU %VEC(1), VEC_SIZE(%rdi) > - VMOVU %VEC(2), -VEC_SIZE(%rdi, %rdx) > - VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi, %rdx) > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(2) > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(3) > + VMOVU %VMM(0), (%rdi) > + VMOVU %VMM(1), VEC_SIZE(%rdi) > + VMOVU %VMM(2), -VEC_SIZE(%rdi, %rdx) > + VMOVU %VMM(3), -(VEC_SIZE * 2)(%rdi, %rdx) > VZEROUPPER_RETURN > > .p2align 4 > @@ -400,24 +392,24 @@ L(more_2x_vec): > cmpq $(VEC_SIZE * 8), %rdx > ja L(more_8x_vec) > /* Load VEC(1) regardless. VEC(0) has already been loaded. */ > - VMOVU VEC_SIZE(%rsi), %VEC(1) > + VMOVU VEC_SIZE(%rsi), %VMM(1) > cmpq $(VEC_SIZE * 4), %rdx > jbe L(last_4x_vec) > /* Copy from 4 * VEC + 1 to 8 * VEC, inclusively. */ > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(4) > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(5) > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(6) > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(7) > - VMOVU %VEC(0), (%rdi) > - VMOVU %VEC(1), VEC_SIZE(%rdi) > - VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) > - VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) > - VMOVU %VEC(4), -VEC_SIZE(%rdi, %rdx) > - VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi, %rdx) > - VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi, %rdx) > - VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi, %rdx) > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(4) > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(5) > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(6) > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(7) > + VMOVU %VMM(0), (%rdi) > + VMOVU %VMM(1), VEC_SIZE(%rdi) > + VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi) > + VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi) > + VMOVU %VMM(4), -VEC_SIZE(%rdi, %rdx) > + VMOVU %VMM(5), -(VEC_SIZE * 2)(%rdi, %rdx) > + VMOVU %VMM(6), -(VEC_SIZE * 3)(%rdi, %rdx) > + VMOVU %VMM(7), -(VEC_SIZE * 4)(%rdi, %rdx) > VZEROUPPER_RETURN > > .p2align 4,, 4 > @@ -466,14 +458,14 @@ L(more_8x_vec_forward): > */ > > /* First vec was already loaded into VEC(0). */ > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5) > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6) > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(5) > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(6) > /* Save begining of dst. */ > movq %rdi, %rcx > /* Align dst to VEC_SIZE - 1. */ > orq $(VEC_SIZE - 1), %rdi > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7) > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8) > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(7) > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(8) > > /* Subtract dst from src. Add back after dst aligned. */ > subq %rcx, %rsi > @@ -488,25 +480,25 @@ L(more_8x_vec_forward): > .p2align 4,, 11 > L(loop_4x_vec_forward): > /* Copy 4 * VEC a time forward. */ > - VMOVU (%rsi), %VEC(1) > - VMOVU VEC_SIZE(%rsi), %VEC(2) > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(3) > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(4) > + VMOVU (%rsi), %VMM(1) > + VMOVU VEC_SIZE(%rsi), %VMM(2) > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(3) > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(4) > subq $-(VEC_SIZE * 4), %rsi > - VMOVA %VEC(1), (%rdi) > - VMOVA %VEC(2), VEC_SIZE(%rdi) > - VMOVA %VEC(3), (VEC_SIZE * 2)(%rdi) > - VMOVA %VEC(4), (VEC_SIZE * 3)(%rdi) > + VMOVA %VMM(1), (%rdi) > + VMOVA %VMM(2), VEC_SIZE(%rdi) > + VMOVA %VMM(3), (VEC_SIZE * 2)(%rdi) > + VMOVA %VMM(4), (VEC_SIZE * 3)(%rdi) > subq $-(VEC_SIZE * 4), %rdi > cmpq %rdi, %rdx > ja L(loop_4x_vec_forward) > /* Store the last 4 * VEC. */ > - VMOVU %VEC(5), (VEC_SIZE * 3)(%rdx) > - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdx) > - VMOVU %VEC(7), VEC_SIZE(%rdx) > - VMOVU %VEC(8), (%rdx) > + VMOVU %VMM(5), (VEC_SIZE * 3)(%rdx) > + VMOVU %VMM(6), (VEC_SIZE * 2)(%rdx) > + VMOVU %VMM(7), VEC_SIZE(%rdx) > + VMOVU %VMM(8), (%rdx) > /* Store the first VEC. */ > - VMOVU %VEC(0), (%rcx) > + VMOVU %VMM(0), (%rcx) > /* Keep L(nop_backward) target close to jmp for 2-byte encoding. > */ > L(nop_backward): > @@ -523,12 +515,12 @@ L(more_8x_vec_backward): > addresses. */ > > /* First vec was also loaded into VEC(0). */ > - VMOVU VEC_SIZE(%rsi), %VEC(5) > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6) > + VMOVU VEC_SIZE(%rsi), %VMM(5) > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(6) > /* Begining of region for 4x backward copy stored in rcx. */ > leaq (VEC_SIZE * -4 + -1)(%rdi, %rdx), %rcx > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7) > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(8) > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(7) > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(8) > /* Subtract dst from src. Add back after dst aligned. */ > subq %rdi, %rsi > /* Align dst. */ > @@ -540,25 +532,25 @@ L(more_8x_vec_backward): > .p2align 4,, 11 > L(loop_4x_vec_backward): > /* Copy 4 * VEC a time backward. */ > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(1) > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > - VMOVU (VEC_SIZE * 1)(%rsi), %VEC(3) > - VMOVU (VEC_SIZE * 0)(%rsi), %VEC(4) > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(1) > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > + VMOVU (VEC_SIZE * 1)(%rsi), %VMM(3) > + VMOVU (VEC_SIZE * 0)(%rsi), %VMM(4) > addq $(VEC_SIZE * -4), %rsi > - VMOVA %VEC(1), (VEC_SIZE * 3)(%rcx) > - VMOVA %VEC(2), (VEC_SIZE * 2)(%rcx) > - VMOVA %VEC(3), (VEC_SIZE * 1)(%rcx) > - VMOVA %VEC(4), (VEC_SIZE * 0)(%rcx) > + VMOVA %VMM(1), (VEC_SIZE * 3)(%rcx) > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rcx) > + VMOVA %VMM(3), (VEC_SIZE * 1)(%rcx) > + VMOVA %VMM(4), (VEC_SIZE * 0)(%rcx) > addq $(VEC_SIZE * -4), %rcx > cmpq %rcx, %rdi > jb L(loop_4x_vec_backward) > /* Store the first 4 * VEC. */ > - VMOVU %VEC(0), (%rdi) > - VMOVU %VEC(5), VEC_SIZE(%rdi) > - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) > - VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) > + VMOVU %VMM(0), (%rdi) > + VMOVU %VMM(5), VEC_SIZE(%rdi) > + VMOVU %VMM(6), (VEC_SIZE * 2)(%rdi) > + VMOVU %VMM(7), (VEC_SIZE * 3)(%rdi) > /* Store the last VEC. */ > - VMOVU %VEC(8), -VEC_SIZE(%rdx, %rdi) > + VMOVU %VMM(8), -VEC_SIZE(%rdx, %rdi) > VZEROUPPER_RETURN > > #if defined USE_MULTIARCH && IS_IN (libc) > @@ -568,7 +560,7 @@ L(loop_4x_vec_backward): > # if ALIGN_MOVSB > L(skip_short_movsb_check): > # if MOVSB_ALIGN_TO > VEC_SIZE > - VMOVU VEC_SIZE(%rsi), %VEC(1) > + VMOVU VEC_SIZE(%rsi), %VMM(1) > # endif > # if MOVSB_ALIGN_TO > (VEC_SIZE * 2) > # error Unsupported MOVSB_ALIGN_TO > @@ -597,9 +589,9 @@ L(skip_short_movsb_check): > > rep movsb > > - VMOVU %VEC(0), (%r8) > + VMOVU %VMM(0), (%r8) > # if MOVSB_ALIGN_TO > VEC_SIZE > - VMOVU %VEC(1), VEC_SIZE(%r8) > + VMOVU %VMM(1), VEC_SIZE(%r8) > # endif > VZEROUPPER_RETURN > # endif > @@ -640,7 +632,7 @@ L(movsb): > # endif > # if ALIGN_MOVSB > # if MOVSB_ALIGN_TO > VEC_SIZE > - VMOVU VEC_SIZE(%rsi), %VEC(1) > + VMOVU VEC_SIZE(%rsi), %VMM(1) > # endif > # if MOVSB_ALIGN_TO > (VEC_SIZE * 2) > # error Unsupported MOVSB_ALIGN_TO > @@ -664,9 +656,9 @@ L(movsb_align_dst): > rep movsb > > /* Store VECs loaded for aligning. */ > - VMOVU %VEC(0), (%r8) > + VMOVU %VMM(0), (%r8) > # if MOVSB_ALIGN_TO > VEC_SIZE > - VMOVU %VEC(1), VEC_SIZE(%r8) > + VMOVU %VMM(1), VEC_SIZE(%r8) > # endif > VZEROUPPER_RETURN > # else /* !ALIGN_MOVSB. */ > @@ -701,18 +693,18 @@ L(large_memcpy_2x): > > /* First vec was also loaded into VEC(0). */ > # if VEC_SIZE < 64 > - VMOVU VEC_SIZE(%rsi), %VEC(1) > + VMOVU VEC_SIZE(%rsi), %VMM(1) > # if VEC_SIZE < 32 > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > # endif > # endif > - VMOVU %VEC(0), (%rdi) > + VMOVU %VMM(0), (%rdi) > # if VEC_SIZE < 64 > - VMOVU %VEC(1), VEC_SIZE(%rdi) > + VMOVU %VMM(1), VEC_SIZE(%rdi) > # if VEC_SIZE < 32 > - VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) > - VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) > + VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi) > + VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi) > # endif > # endif > > @@ -761,12 +753,12 @@ L(loop_large_memcpy_2x_inner): > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE) > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE * 2) > /* Load vectors from rsi. */ > - LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > + LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > + LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > subq $-LARGE_LOAD_SIZE, %rsi > /* Non-temporal store vectors to rdi. */ > - STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > - STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > + STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > + STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > subq $-LARGE_LOAD_SIZE, %rdi > decl %ecx > jnz L(loop_large_memcpy_2x_inner) > @@ -785,31 +777,31 @@ L(loop_large_memcpy_2x_tail): > /* Copy 4 * VEC a time forward with non-temporal stores. */ > PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) > PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) > - VMOVU (%rsi), %VEC(0) > - VMOVU VEC_SIZE(%rsi), %VEC(1) > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > + VMOVU (%rsi), %VMM(0) > + VMOVU VEC_SIZE(%rsi), %VMM(1) > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > subq $-(VEC_SIZE * 4), %rsi > addl $-(VEC_SIZE * 4), %edx > - VMOVA %VEC(0), (%rdi) > - VMOVA %VEC(1), VEC_SIZE(%rdi) > - VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) > - VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) > + VMOVA %VMM(0), (%rdi) > + VMOVA %VMM(1), VEC_SIZE(%rdi) > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi) > + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi) > subq $-(VEC_SIZE * 4), %rdi > cmpl $(VEC_SIZE * 4), %edx > ja L(loop_large_memcpy_2x_tail) > > L(large_memcpy_2x_end): > /* Store the last 4 * VEC. */ > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) > - > - VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) > - VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) > - VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) > - VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0) > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1) > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2) > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3) > + > + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx) > + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx) > + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx) > + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx) > VZEROUPPER_RETURN > > .p2align 4 > @@ -831,16 +823,16 @@ L(loop_large_memcpy_4x_inner): > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE) > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 3 + PREFETCHED_LOAD_SIZE) > /* Load vectors from rsi. */ > - LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > - LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) > - LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) > + LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > + LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > + LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11)) > + LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15)) > subq $-LARGE_LOAD_SIZE, %rsi > /* Non-temporal store vectors to rdi. */ > - STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > - STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > - STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) > - STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) > + STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > + STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > + STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11)) > + STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15)) > subq $-LARGE_LOAD_SIZE, %rdi > decl %ecx > jnz L(loop_large_memcpy_4x_inner) > @@ -858,31 +850,31 @@ L(loop_large_memcpy_4x_tail): > /* Copy 4 * VEC a time forward with non-temporal stores. */ > PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) > PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) > - VMOVU (%rsi), %VEC(0) > - VMOVU VEC_SIZE(%rsi), %VEC(1) > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > + VMOVU (%rsi), %VMM(0) > + VMOVU VEC_SIZE(%rsi), %VMM(1) > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > subq $-(VEC_SIZE * 4), %rsi > addl $-(VEC_SIZE * 4), %edx > - VMOVA %VEC(0), (%rdi) > - VMOVA %VEC(1), VEC_SIZE(%rdi) > - VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) > - VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) > + VMOVA %VMM(0), (%rdi) > + VMOVA %VMM(1), VEC_SIZE(%rdi) > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi) > + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi) > subq $-(VEC_SIZE * 4), %rdi > cmpl $(VEC_SIZE * 4), %edx > ja L(loop_large_memcpy_4x_tail) > > L(large_memcpy_4x_end): > /* Store the last 4 * VEC. */ > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) > - > - VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) > - VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) > - VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) > - VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0) > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1) > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2) > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3) > + > + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx) > + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx) > + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx) > + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx) > VZEROUPPER_RETURN > #endif > END (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) > -- > 2.34.1 >
On Fri, Oct 14, 2022 at 9:53 PM H.J. Lu <hjl.tools@gmail.com> wrote: > > On Fri, Oct 14, 2022 at 5:21 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > > > Replace %VEC(n) -> %VMM(n) > > > > This commit does not change libc.so > > > > Tested build on x86-64 > > --- > > .../memmove-avx-unaligned-erms-rtm.S | 15 +- > > .../multiarch/memmove-avx-unaligned-erms.S | 9 +- > > .../multiarch/memmove-avx512-unaligned-erms.S | 30 +- > > .../multiarch/memmove-evex-unaligned-erms.S | 30 +- > > .../multiarch/memmove-sse2-unaligned-erms.S | 11 +- > > .../multiarch/memmove-vec-unaligned-erms.S | 262 +++++++++--------- > > 6 files changed, 135 insertions(+), 222 deletions(-) > > > > diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S > > index 67a55f0c85..c2a95dc247 100644 > > --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S > > +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S > > @@ -1,16 +1,9 @@ > > -#if IS_IN (libc) > > Keep this. > > > -# define VEC_SIZE 32 > > -# define VEC(i) ymm##i > > -# define VMOVNT vmovntdq > > -# define VMOVU vmovdqu > > -# define VMOVA vmovdqa > > -# define MOV_SIZE 4 > > -# define ZERO_UPPER_VEC_REGISTERS_RETURN \ > > - ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST > > +#include <isa-level.h> > > > > -# define VZEROUPPER_RETURN jmp L(return) > > +#if ISA_SHOULD_BUILD (3) > > This ISA_SHOULD_BUILD change isn't needed. kk will fix for next version. Although I think we should patch that. > > > +# include "x86-avx-rtm-vecs.h" > > > > -# define SECTION(p) p##.avx.rtm > > # define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_rtm > > > > # include "memmove-vec-unaligned-erms.S" > > diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S > > index a14b155667..4e4b4635f9 100644 > > --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S > > @@ -2,14 +2,7 @@ > > > > #if ISA_SHOULD_BUILD (3) > > > > -# define VEC_SIZE 32 > > -# define VEC(i) ymm##i > > -# define VMOVNT vmovntdq > > -# define VMOVU vmovdqu > > -# define VMOVA vmovdqa > > -# define MOV_SIZE 4 > > - > > -# define SECTION(p) p##.avx > > +# include "x86-avx-vecs.h" > > > > # ifndef MEMMOVE_SYMBOL > > # define MEMMOVE_SYMBOL(p,s) p##_avx_##s > > diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S > > index 8d1568a7ba..cca97e38f8 100644 > > --- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S > > @@ -2,35 +2,7 @@ > > > > #if ISA_SHOULD_BUILD (4) > > > > -# define VEC_SIZE 64 > > -# define XMM0 xmm16 > > -# define XMM1 xmm17 > > -# define YMM0 ymm16 > > -# define YMM1 ymm17 > > -# define VEC0 zmm16 > > -# define VEC1 zmm17 > > -# define VEC2 zmm18 > > -# define VEC3 zmm19 > > -# define VEC4 zmm20 > > -# define VEC5 zmm21 > > -# define VEC6 zmm22 > > -# define VEC7 zmm23 > > -# define VEC8 zmm24 > > -# define VEC9 zmm25 > > -# define VEC10 zmm26 > > -# define VEC11 zmm27 > > -# define VEC12 zmm28 > > -# define VEC13 zmm29 > > -# define VEC14 zmm30 > > -# define VEC15 zmm31 > > -# define VEC(i) VEC##i > > -# define VMOVNT vmovntdq > > -# define VMOVU vmovdqu64 > > -# define VMOVA vmovdqa64 > > -# define VZEROUPPER > > -# define MOV_SIZE 6 > > - > > -# define SECTION(p) p##.evex512 > > +# include "x86-evex512-vecs.h" > > > > # ifndef MEMMOVE_SYMBOL > > # define MEMMOVE_SYMBOL(p,s) p##_avx512_##s > > diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S > > index 2373017358..1f7b5715f7 100644 > > --- a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S > > @@ -2,35 +2,7 @@ > > > > #if ISA_SHOULD_BUILD (4) > > > > -# define VEC_SIZE 32 > > -# define XMM0 xmm16 > > -# define XMM1 xmm17 > > -# define YMM0 ymm16 > > -# define YMM1 ymm17 > > -# define VEC0 ymm16 > > -# define VEC1 ymm17 > > -# define VEC2 ymm18 > > -# define VEC3 ymm19 > > -# define VEC4 ymm20 > > -# define VEC5 ymm21 > > -# define VEC6 ymm22 > > -# define VEC7 ymm23 > > -# define VEC8 ymm24 > > -# define VEC9 ymm25 > > -# define VEC10 ymm26 > > -# define VEC11 ymm27 > > -# define VEC12 ymm28 > > -# define VEC13 ymm29 > > -# define VEC14 ymm30 > > -# define VEC15 ymm31 > > -# define VEC(i) VEC##i > > -# define VMOVNT vmovntdq > > -# define VMOVU vmovdqu64 > > -# define VMOVA vmovdqa64 > > -# define VZEROUPPER > > -# define MOV_SIZE 6 > > - > > -# define SECTION(p) p##.evex > > +# include "x86-evex256-vecs.h" > > > > # ifndef MEMMOVE_SYMBOL > > # define MEMMOVE_SYMBOL(p,s) p##_evex_##s > > diff --git a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S > > index 422a079902..8431bcd000 100644 > > --- a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S > > @@ -22,18 +22,9 @@ > > so we need this to build for ISA V2 builds. */ > > #if ISA_SHOULD_BUILD (2) > > > > -# include <sysdep.h> > > +# include "x86-sse2-vecs.h" > > > > -# define VEC_SIZE 16 > > -# define VEC(i) xmm##i > > # define PREFETCHNT prefetchnta > > -# define VMOVNT movntdq > > -/* Use movups and movaps for smaller code sizes. */ > > -# define VMOVU movups > > -# define VMOVA movaps > > -# define MOV_SIZE 3 > > - > > -# define SECTION(p) p > > > > # ifndef MEMMOVE_SYMBOL > > # define MEMMOVE_SYMBOL(p,s) p##_sse2_##s > > diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S > > index 04747133b7..5b758cae5e 100644 > > --- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S > > +++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S > > @@ -60,14 +60,6 @@ > > # define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) > > #endif > > > > -#ifndef XMM0 > > -# define XMM0 xmm0 > > -#endif > > - > > -#ifndef YMM0 > > -# define YMM0 ymm0 > > -#endif > > - > > #ifndef VZEROUPPER > > # if VEC_SIZE > 16 > > # define VZEROUPPER vzeroupper > > @@ -225,13 +217,13 @@ L(start): > > cmp $VEC_SIZE, %RDX_LP > > jb L(less_vec) > > /* Load regardless. */ > > - VMOVU (%rsi), %VEC(0) > > + VMOVU (%rsi), %VMM(0) > > cmp $(VEC_SIZE * 2), %RDX_LP > > ja L(more_2x_vec) > > /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ > > - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) > > - VMOVU %VEC(0), (%rdi) > > - VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) > > + VMOVU -VEC_SIZE(%rsi,%rdx), %VMM(1) > > + VMOVU %VMM(0), (%rdi) > > + VMOVU %VMM(1), -VEC_SIZE(%rdi,%rdx) > > #if !(defined USE_MULTIARCH && IS_IN (libc)) > > ZERO_UPPER_VEC_REGISTERS_RETURN > > #else > > @@ -270,15 +262,15 @@ L(start_erms): > > cmp $VEC_SIZE, %RDX_LP > > jb L(less_vec) > > /* Load regardless. */ > > - VMOVU (%rsi), %VEC(0) > > + VMOVU (%rsi), %VMM(0) > > cmp $(VEC_SIZE * 2), %RDX_LP > > ja L(movsb_more_2x_vec) > > /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. > > */ > > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(1) > > - VMOVU %VEC(0), (%rdi) > > - VMOVU %VEC(1), -VEC_SIZE(%rdi, %rdx) > > -L(return): > > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(1) > > + VMOVU %VMM(0), (%rdi) > > + VMOVU %VMM(1), -VEC_SIZE(%rdi, %rdx) > > +L(return_vzeroupper): > > # if VEC_SIZE > 16 > > ZERO_UPPER_VEC_REGISTERS_RETURN > > # else > > @@ -359,10 +351,10 @@ L(between_16_31): > > .p2align 4,, 10 > > L(between_32_63): > > /* From 32 to 63. No branch when size == 32. */ > > - VMOVU (%rsi), %YMM0 > > - VMOVU -32(%rsi, %rdx), %YMM1 > > - VMOVU %YMM0, (%rdi) > > - VMOVU %YMM1, -32(%rdi, %rdx) > > + VMOVU (%rsi), %VMM_256(0) > > + VMOVU -32(%rsi, %rdx), %VMM_256(1) > > + VMOVU %VMM_256(0), (%rdi) > > + VMOVU %VMM_256(1), -32(%rdi, %rdx) > > VZEROUPPER_RETURN > > #endif > > > > @@ -380,12 +372,12 @@ L(last_4x_vec): > > /* Copy from 2 * VEC + 1 to 4 * VEC, inclusively. */ > > > > /* VEC(0) and VEC(1) have already been loaded. */ > > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(2) > > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(3) > > - VMOVU %VEC(0), (%rdi) > > - VMOVU %VEC(1), VEC_SIZE(%rdi) > > - VMOVU %VEC(2), -VEC_SIZE(%rdi, %rdx) > > - VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi, %rdx) > > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(2) > > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(3) > > + VMOVU %VMM(0), (%rdi) > > + VMOVU %VMM(1), VEC_SIZE(%rdi) > > + VMOVU %VMM(2), -VEC_SIZE(%rdi, %rdx) > > + VMOVU %VMM(3), -(VEC_SIZE * 2)(%rdi, %rdx) > > VZEROUPPER_RETURN > > > > .p2align 4 > > @@ -400,24 +392,24 @@ L(more_2x_vec): > > cmpq $(VEC_SIZE * 8), %rdx > > ja L(more_8x_vec) > > /* Load VEC(1) regardless. VEC(0) has already been loaded. */ > > - VMOVU VEC_SIZE(%rsi), %VEC(1) > > + VMOVU VEC_SIZE(%rsi), %VMM(1) > > cmpq $(VEC_SIZE * 4), %rdx > > jbe L(last_4x_vec) > > /* Copy from 4 * VEC + 1 to 8 * VEC, inclusively. */ > > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(4) > > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(5) > > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(6) > > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(7) > > - VMOVU %VEC(0), (%rdi) > > - VMOVU %VEC(1), VEC_SIZE(%rdi) > > - VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) > > - VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) > > - VMOVU %VEC(4), -VEC_SIZE(%rdi, %rdx) > > - VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi, %rdx) > > - VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi, %rdx) > > - VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi, %rdx) > > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(4) > > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(5) > > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(6) > > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(7) > > + VMOVU %VMM(0), (%rdi) > > + VMOVU %VMM(1), VEC_SIZE(%rdi) > > + VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi) > > + VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi) > > + VMOVU %VMM(4), -VEC_SIZE(%rdi, %rdx) > > + VMOVU %VMM(5), -(VEC_SIZE * 2)(%rdi, %rdx) > > + VMOVU %VMM(6), -(VEC_SIZE * 3)(%rdi, %rdx) > > + VMOVU %VMM(7), -(VEC_SIZE * 4)(%rdi, %rdx) > > VZEROUPPER_RETURN > > > > .p2align 4,, 4 > > @@ -466,14 +458,14 @@ L(more_8x_vec_forward): > > */ > > > > /* First vec was already loaded into VEC(0). */ > > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5) > > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6) > > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(5) > > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(6) > > /* Save begining of dst. */ > > movq %rdi, %rcx > > /* Align dst to VEC_SIZE - 1. */ > > orq $(VEC_SIZE - 1), %rdi > > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7) > > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8) > > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(7) > > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(8) > > > > /* Subtract dst from src. Add back after dst aligned. */ > > subq %rcx, %rsi > > @@ -488,25 +480,25 @@ L(more_8x_vec_forward): > > .p2align 4,, 11 > > L(loop_4x_vec_forward): > > /* Copy 4 * VEC a time forward. */ > > - VMOVU (%rsi), %VEC(1) > > - VMOVU VEC_SIZE(%rsi), %VEC(2) > > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(3) > > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(4) > > + VMOVU (%rsi), %VMM(1) > > + VMOVU VEC_SIZE(%rsi), %VMM(2) > > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(3) > > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(4) > > subq $-(VEC_SIZE * 4), %rsi > > - VMOVA %VEC(1), (%rdi) > > - VMOVA %VEC(2), VEC_SIZE(%rdi) > > - VMOVA %VEC(3), (VEC_SIZE * 2)(%rdi) > > - VMOVA %VEC(4), (VEC_SIZE * 3)(%rdi) > > + VMOVA %VMM(1), (%rdi) > > + VMOVA %VMM(2), VEC_SIZE(%rdi) > > + VMOVA %VMM(3), (VEC_SIZE * 2)(%rdi) > > + VMOVA %VMM(4), (VEC_SIZE * 3)(%rdi) > > subq $-(VEC_SIZE * 4), %rdi > > cmpq %rdi, %rdx > > ja L(loop_4x_vec_forward) > > /* Store the last 4 * VEC. */ > > - VMOVU %VEC(5), (VEC_SIZE * 3)(%rdx) > > - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdx) > > - VMOVU %VEC(7), VEC_SIZE(%rdx) > > - VMOVU %VEC(8), (%rdx) > > + VMOVU %VMM(5), (VEC_SIZE * 3)(%rdx) > > + VMOVU %VMM(6), (VEC_SIZE * 2)(%rdx) > > + VMOVU %VMM(7), VEC_SIZE(%rdx) > > + VMOVU %VMM(8), (%rdx) > > /* Store the first VEC. */ > > - VMOVU %VEC(0), (%rcx) > > + VMOVU %VMM(0), (%rcx) > > /* Keep L(nop_backward) target close to jmp for 2-byte encoding. > > */ > > L(nop_backward): > > @@ -523,12 +515,12 @@ L(more_8x_vec_backward): > > addresses. */ > > > > /* First vec was also loaded into VEC(0). */ > > - VMOVU VEC_SIZE(%rsi), %VEC(5) > > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6) > > + VMOVU VEC_SIZE(%rsi), %VMM(5) > > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(6) > > /* Begining of region for 4x backward copy stored in rcx. */ > > leaq (VEC_SIZE * -4 + -1)(%rdi, %rdx), %rcx > > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7) > > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(8) > > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(7) > > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(8) > > /* Subtract dst from src. Add back after dst aligned. */ > > subq %rdi, %rsi > > /* Align dst. */ > > @@ -540,25 +532,25 @@ L(more_8x_vec_backward): > > .p2align 4,, 11 > > L(loop_4x_vec_backward): > > /* Copy 4 * VEC a time backward. */ > > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(1) > > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > > - VMOVU (VEC_SIZE * 1)(%rsi), %VEC(3) > > - VMOVU (VEC_SIZE * 0)(%rsi), %VEC(4) > > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(1) > > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > > + VMOVU (VEC_SIZE * 1)(%rsi), %VMM(3) > > + VMOVU (VEC_SIZE * 0)(%rsi), %VMM(4) > > addq $(VEC_SIZE * -4), %rsi > > - VMOVA %VEC(1), (VEC_SIZE * 3)(%rcx) > > - VMOVA %VEC(2), (VEC_SIZE * 2)(%rcx) > > - VMOVA %VEC(3), (VEC_SIZE * 1)(%rcx) > > - VMOVA %VEC(4), (VEC_SIZE * 0)(%rcx) > > + VMOVA %VMM(1), (VEC_SIZE * 3)(%rcx) > > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rcx) > > + VMOVA %VMM(3), (VEC_SIZE * 1)(%rcx) > > + VMOVA %VMM(4), (VEC_SIZE * 0)(%rcx) > > addq $(VEC_SIZE * -4), %rcx > > cmpq %rcx, %rdi > > jb L(loop_4x_vec_backward) > > /* Store the first 4 * VEC. */ > > - VMOVU %VEC(0), (%rdi) > > - VMOVU %VEC(5), VEC_SIZE(%rdi) > > - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) > > - VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) > > + VMOVU %VMM(0), (%rdi) > > + VMOVU %VMM(5), VEC_SIZE(%rdi) > > + VMOVU %VMM(6), (VEC_SIZE * 2)(%rdi) > > + VMOVU %VMM(7), (VEC_SIZE * 3)(%rdi) > > /* Store the last VEC. */ > > - VMOVU %VEC(8), -VEC_SIZE(%rdx, %rdi) > > + VMOVU %VMM(8), -VEC_SIZE(%rdx, %rdi) > > VZEROUPPER_RETURN > > > > #if defined USE_MULTIARCH && IS_IN (libc) > > @@ -568,7 +560,7 @@ L(loop_4x_vec_backward): > > # if ALIGN_MOVSB > > L(skip_short_movsb_check): > > # if MOVSB_ALIGN_TO > VEC_SIZE > > - VMOVU VEC_SIZE(%rsi), %VEC(1) > > + VMOVU VEC_SIZE(%rsi), %VMM(1) > > # endif > > # if MOVSB_ALIGN_TO > (VEC_SIZE * 2) > > # error Unsupported MOVSB_ALIGN_TO > > @@ -597,9 +589,9 @@ L(skip_short_movsb_check): > > > > rep movsb > > > > - VMOVU %VEC(0), (%r8) > > + VMOVU %VMM(0), (%r8) > > # if MOVSB_ALIGN_TO > VEC_SIZE > > - VMOVU %VEC(1), VEC_SIZE(%r8) > > + VMOVU %VMM(1), VEC_SIZE(%r8) > > # endif > > VZEROUPPER_RETURN > > # endif > > @@ -640,7 +632,7 @@ L(movsb): > > # endif > > # if ALIGN_MOVSB > > # if MOVSB_ALIGN_TO > VEC_SIZE > > - VMOVU VEC_SIZE(%rsi), %VEC(1) > > + VMOVU VEC_SIZE(%rsi), %VMM(1) > > # endif > > # if MOVSB_ALIGN_TO > (VEC_SIZE * 2) > > # error Unsupported MOVSB_ALIGN_TO > > @@ -664,9 +656,9 @@ L(movsb_align_dst): > > rep movsb > > > > /* Store VECs loaded for aligning. */ > > - VMOVU %VEC(0), (%r8) > > + VMOVU %VMM(0), (%r8) > > # if MOVSB_ALIGN_TO > VEC_SIZE > > - VMOVU %VEC(1), VEC_SIZE(%r8) > > + VMOVU %VMM(1), VEC_SIZE(%r8) > > # endif > > VZEROUPPER_RETURN > > # else /* !ALIGN_MOVSB. */ > > @@ -701,18 +693,18 @@ L(large_memcpy_2x): > > > > /* First vec was also loaded into VEC(0). */ > > # if VEC_SIZE < 64 > > - VMOVU VEC_SIZE(%rsi), %VEC(1) > > + VMOVU VEC_SIZE(%rsi), %VMM(1) > > # if VEC_SIZE < 32 > > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > > # endif > > # endif > > - VMOVU %VEC(0), (%rdi) > > + VMOVU %VMM(0), (%rdi) > > # if VEC_SIZE < 64 > > - VMOVU %VEC(1), VEC_SIZE(%rdi) > > + VMOVU %VMM(1), VEC_SIZE(%rdi) > > # if VEC_SIZE < 32 > > - VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) > > - VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) > > + VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi) > > + VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi) > > # endif > > # endif > > > > @@ -761,12 +753,12 @@ L(loop_large_memcpy_2x_inner): > > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE) > > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE * 2) > > /* Load vectors from rsi. */ > > - LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > > - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > > + LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > > + LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > > subq $-LARGE_LOAD_SIZE, %rsi > > /* Non-temporal store vectors to rdi. */ > > - STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > > - STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > > + STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > > + STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > > subq $-LARGE_LOAD_SIZE, %rdi > > decl %ecx > > jnz L(loop_large_memcpy_2x_inner) > > @@ -785,31 +777,31 @@ L(loop_large_memcpy_2x_tail): > > /* Copy 4 * VEC a time forward with non-temporal stores. */ > > PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) > > PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) > > - VMOVU (%rsi), %VEC(0) > > - VMOVU VEC_SIZE(%rsi), %VEC(1) > > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > > + VMOVU (%rsi), %VMM(0) > > + VMOVU VEC_SIZE(%rsi), %VMM(1) > > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > > subq $-(VEC_SIZE * 4), %rsi > > addl $-(VEC_SIZE * 4), %edx > > - VMOVA %VEC(0), (%rdi) > > - VMOVA %VEC(1), VEC_SIZE(%rdi) > > - VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) > > - VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) > > + VMOVA %VMM(0), (%rdi) > > + VMOVA %VMM(1), VEC_SIZE(%rdi) > > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi) > > + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi) > > subq $-(VEC_SIZE * 4), %rdi > > cmpl $(VEC_SIZE * 4), %edx > > ja L(loop_large_memcpy_2x_tail) > > > > L(large_memcpy_2x_end): > > /* Store the last 4 * VEC. */ > > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) > > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) > > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) > > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) > > - > > - VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) > > - VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) > > - VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) > > - VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) > > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0) > > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1) > > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2) > > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3) > > + > > + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx) > > + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx) > > + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx) > > + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx) > > VZEROUPPER_RETURN > > > > .p2align 4 > > @@ -831,16 +823,16 @@ L(loop_large_memcpy_4x_inner): > > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE) > > PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 3 + PREFETCHED_LOAD_SIZE) > > /* Load vectors from rsi. */ > > - LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > > - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > > - LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) > > - LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) > > + LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > > + LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > > + LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11)) > > + LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15)) > > subq $-LARGE_LOAD_SIZE, %rsi > > /* Non-temporal store vectors to rdi. */ > > - STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) > > - STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) > > - STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) > > - STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) > > + STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) > > + STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) > > + STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11)) > > + STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15)) > > subq $-LARGE_LOAD_SIZE, %rdi > > decl %ecx > > jnz L(loop_large_memcpy_4x_inner) > > @@ -858,31 +850,31 @@ L(loop_large_memcpy_4x_tail): > > /* Copy 4 * VEC a time forward with non-temporal stores. */ > > PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) > > PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) > > - VMOVU (%rsi), %VEC(0) > > - VMOVU VEC_SIZE(%rsi), %VEC(1) > > - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) > > - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) > > + VMOVU (%rsi), %VMM(0) > > + VMOVU VEC_SIZE(%rsi), %VMM(1) > > + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) > > + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) > > subq $-(VEC_SIZE * 4), %rsi > > addl $-(VEC_SIZE * 4), %edx > > - VMOVA %VEC(0), (%rdi) > > - VMOVA %VEC(1), VEC_SIZE(%rdi) > > - VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) > > - VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) > > + VMOVA %VMM(0), (%rdi) > > + VMOVA %VMM(1), VEC_SIZE(%rdi) > > + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi) > > + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi) > > subq $-(VEC_SIZE * 4), %rdi > > cmpl $(VEC_SIZE * 4), %edx > > ja L(loop_large_memcpy_4x_tail) > > > > L(large_memcpy_4x_end): > > /* Store the last 4 * VEC. */ > > - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) > > - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) > > - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) > > - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) > > - > > - VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) > > - VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) > > - VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) > > - VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) > > + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0) > > + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1) > > + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2) > > + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3) > > + > > + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx) > > + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx) > > + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx) > > + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx) > > VZEROUPPER_RETURN > > #endif > > END (MEMMOVE_SYMBOL (__memmove, unaligned_erms)) > > -- > > 2.34.1 > > > > > -- > H.J.
diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S index 67a55f0c85..c2a95dc247 100644 --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms-rtm.S @@ -1,16 +1,9 @@ -#if IS_IN (libc) -# define VEC_SIZE 32 -# define VEC(i) ymm##i -# define VMOVNT vmovntdq -# define VMOVU vmovdqu -# define VMOVA vmovdqa -# define MOV_SIZE 4 -# define ZERO_UPPER_VEC_REGISTERS_RETURN \ - ZERO_UPPER_VEC_REGISTERS_RETURN_XTEST +#include <isa-level.h> -# define VZEROUPPER_RETURN jmp L(return) +#if ISA_SHOULD_BUILD (3) + +# include "x86-avx-rtm-vecs.h" -# define SECTION(p) p##.avx.rtm # define MEMMOVE_SYMBOL(p,s) p##_avx_##s##_rtm # include "memmove-vec-unaligned-erms.S" diff --git a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S index a14b155667..4e4b4635f9 100644 --- a/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S +++ b/sysdeps/x86_64/multiarch/memmove-avx-unaligned-erms.S @@ -2,14 +2,7 @@ #if ISA_SHOULD_BUILD (3) -# define VEC_SIZE 32 -# define VEC(i) ymm##i -# define VMOVNT vmovntdq -# define VMOVU vmovdqu -# define VMOVA vmovdqa -# define MOV_SIZE 4 - -# define SECTION(p) p##.avx +# include "x86-avx-vecs.h" # ifndef MEMMOVE_SYMBOL # define MEMMOVE_SYMBOL(p,s) p##_avx_##s diff --git a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S index 8d1568a7ba..cca97e38f8 100644 --- a/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S +++ b/sysdeps/x86_64/multiarch/memmove-avx512-unaligned-erms.S @@ -2,35 +2,7 @@ #if ISA_SHOULD_BUILD (4) -# define VEC_SIZE 64 -# define XMM0 xmm16 -# define XMM1 xmm17 -# define YMM0 ymm16 -# define YMM1 ymm17 -# define VEC0 zmm16 -# define VEC1 zmm17 -# define VEC2 zmm18 -# define VEC3 zmm19 -# define VEC4 zmm20 -# define VEC5 zmm21 -# define VEC6 zmm22 -# define VEC7 zmm23 -# define VEC8 zmm24 -# define VEC9 zmm25 -# define VEC10 zmm26 -# define VEC11 zmm27 -# define VEC12 zmm28 -# define VEC13 zmm29 -# define VEC14 zmm30 -# define VEC15 zmm31 -# define VEC(i) VEC##i -# define VMOVNT vmovntdq -# define VMOVU vmovdqu64 -# define VMOVA vmovdqa64 -# define VZEROUPPER -# define MOV_SIZE 6 - -# define SECTION(p) p##.evex512 +# include "x86-evex512-vecs.h" # ifndef MEMMOVE_SYMBOL # define MEMMOVE_SYMBOL(p,s) p##_avx512_##s diff --git a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S index 2373017358..1f7b5715f7 100644 --- a/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S +++ b/sysdeps/x86_64/multiarch/memmove-evex-unaligned-erms.S @@ -2,35 +2,7 @@ #if ISA_SHOULD_BUILD (4) -# define VEC_SIZE 32 -# define XMM0 xmm16 -# define XMM1 xmm17 -# define YMM0 ymm16 -# define YMM1 ymm17 -# define VEC0 ymm16 -# define VEC1 ymm17 -# define VEC2 ymm18 -# define VEC3 ymm19 -# define VEC4 ymm20 -# define VEC5 ymm21 -# define VEC6 ymm22 -# define VEC7 ymm23 -# define VEC8 ymm24 -# define VEC9 ymm25 -# define VEC10 ymm26 -# define VEC11 ymm27 -# define VEC12 ymm28 -# define VEC13 ymm29 -# define VEC14 ymm30 -# define VEC15 ymm31 -# define VEC(i) VEC##i -# define VMOVNT vmovntdq -# define VMOVU vmovdqu64 -# define VMOVA vmovdqa64 -# define VZEROUPPER -# define MOV_SIZE 6 - -# define SECTION(p) p##.evex +# include "x86-evex256-vecs.h" # ifndef MEMMOVE_SYMBOL # define MEMMOVE_SYMBOL(p,s) p##_evex_##s diff --git a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S index 422a079902..8431bcd000 100644 --- a/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S +++ b/sysdeps/x86_64/multiarch/memmove-sse2-unaligned-erms.S @@ -22,18 +22,9 @@ so we need this to build for ISA V2 builds. */ #if ISA_SHOULD_BUILD (2) -# include <sysdep.h> +# include "x86-sse2-vecs.h" -# define VEC_SIZE 16 -# define VEC(i) xmm##i # define PREFETCHNT prefetchnta -# define VMOVNT movntdq -/* Use movups and movaps for smaller code sizes. */ -# define VMOVU movups -# define VMOVA movaps -# define MOV_SIZE 3 - -# define SECTION(p) p # ifndef MEMMOVE_SYMBOL # define MEMMOVE_SYMBOL(p,s) p##_sse2_##s diff --git a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S index 04747133b7..5b758cae5e 100644 --- a/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S +++ b/sysdeps/x86_64/multiarch/memmove-vec-unaligned-erms.S @@ -60,14 +60,6 @@ # define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s) #endif -#ifndef XMM0 -# define XMM0 xmm0 -#endif - -#ifndef YMM0 -# define YMM0 ymm0 -#endif - #ifndef VZEROUPPER # if VEC_SIZE > 16 # define VZEROUPPER vzeroupper @@ -225,13 +217,13 @@ L(start): cmp $VEC_SIZE, %RDX_LP jb L(less_vec) /* Load regardless. */ - VMOVU (%rsi), %VEC(0) + VMOVU (%rsi), %VMM(0) cmp $(VEC_SIZE * 2), %RDX_LP ja L(more_2x_vec) /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ - VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx) + VMOVU -VEC_SIZE(%rsi,%rdx), %VMM(1) + VMOVU %VMM(0), (%rdi) + VMOVU %VMM(1), -VEC_SIZE(%rdi,%rdx) #if !(defined USE_MULTIARCH && IS_IN (libc)) ZERO_UPPER_VEC_REGISTERS_RETURN #else @@ -270,15 +262,15 @@ L(start_erms): cmp $VEC_SIZE, %RDX_LP jb L(less_vec) /* Load regardless. */ - VMOVU (%rsi), %VEC(0) + VMOVU (%rsi), %VMM(0) cmp $(VEC_SIZE * 2), %RDX_LP ja L(movsb_more_2x_vec) /* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */ - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(1) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), -VEC_SIZE(%rdi, %rdx) -L(return): + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(1) + VMOVU %VMM(0), (%rdi) + VMOVU %VMM(1), -VEC_SIZE(%rdi, %rdx) +L(return_vzeroupper): # if VEC_SIZE > 16 ZERO_UPPER_VEC_REGISTERS_RETURN # else @@ -359,10 +351,10 @@ L(between_16_31): .p2align 4,, 10 L(between_32_63): /* From 32 to 63. No branch when size == 32. */ - VMOVU (%rsi), %YMM0 - VMOVU -32(%rsi, %rdx), %YMM1 - VMOVU %YMM0, (%rdi) - VMOVU %YMM1, -32(%rdi, %rdx) + VMOVU (%rsi), %VMM_256(0) + VMOVU -32(%rsi, %rdx), %VMM_256(1) + VMOVU %VMM_256(0), (%rdi) + VMOVU %VMM_256(1), -32(%rdi, %rdx) VZEROUPPER_RETURN #endif @@ -380,12 +372,12 @@ L(last_4x_vec): /* Copy from 2 * VEC + 1 to 4 * VEC, inclusively. */ /* VEC(0) and VEC(1) have already been loaded. */ - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(2) - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(3) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), VEC_SIZE(%rdi) - VMOVU %VEC(2), -VEC_SIZE(%rdi, %rdx) - VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi, %rdx) + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(2) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(3) + VMOVU %VMM(0), (%rdi) + VMOVU %VMM(1), VEC_SIZE(%rdi) + VMOVU %VMM(2), -VEC_SIZE(%rdi, %rdx) + VMOVU %VMM(3), -(VEC_SIZE * 2)(%rdi, %rdx) VZEROUPPER_RETURN .p2align 4 @@ -400,24 +392,24 @@ L(more_2x_vec): cmpq $(VEC_SIZE * 8), %rdx ja L(more_8x_vec) /* Load VEC(1) regardless. VEC(0) has already been loaded. */ - VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU VEC_SIZE(%rsi), %VMM(1) cmpq $(VEC_SIZE * 4), %rdx jbe L(last_4x_vec) /* Copy from 4 * VEC + 1 to 8 * VEC, inclusively. */ - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(4) - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(5) - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(6) - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(7) - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(1), VEC_SIZE(%rdi) - VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) - VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) - VMOVU %VEC(4), -VEC_SIZE(%rdi, %rdx) - VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi, %rdx) - VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi, %rdx) - VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi, %rdx) + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(4) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(5) + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(6) + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(7) + VMOVU %VMM(0), (%rdi) + VMOVU %VMM(1), VEC_SIZE(%rdi) + VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi) + VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi) + VMOVU %VMM(4), -VEC_SIZE(%rdi, %rdx) + VMOVU %VMM(5), -(VEC_SIZE * 2)(%rdi, %rdx) + VMOVU %VMM(6), -(VEC_SIZE * 3)(%rdi, %rdx) + VMOVU %VMM(7), -(VEC_SIZE * 4)(%rdi, %rdx) VZEROUPPER_RETURN .p2align 4,, 4 @@ -466,14 +458,14 @@ L(more_8x_vec_forward): */ /* First vec was already loaded into VEC(0). */ - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5) - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6) + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(5) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(6) /* Save begining of dst. */ movq %rdi, %rcx /* Align dst to VEC_SIZE - 1. */ orq $(VEC_SIZE - 1), %rdi - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7) - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8) + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(7) + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(8) /* Subtract dst from src. Add back after dst aligned. */ subq %rcx, %rsi @@ -488,25 +480,25 @@ L(more_8x_vec_forward): .p2align 4,, 11 L(loop_4x_vec_forward): /* Copy 4 * VEC a time forward. */ - VMOVU (%rsi), %VEC(1) - VMOVU VEC_SIZE(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(3) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(4) + VMOVU (%rsi), %VMM(1) + VMOVU VEC_SIZE(%rsi), %VMM(2) + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(3) + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(4) subq $-(VEC_SIZE * 4), %rsi - VMOVA %VEC(1), (%rdi) - VMOVA %VEC(2), VEC_SIZE(%rdi) - VMOVA %VEC(3), (VEC_SIZE * 2)(%rdi) - VMOVA %VEC(4), (VEC_SIZE * 3)(%rdi) + VMOVA %VMM(1), (%rdi) + VMOVA %VMM(2), VEC_SIZE(%rdi) + VMOVA %VMM(3), (VEC_SIZE * 2)(%rdi) + VMOVA %VMM(4), (VEC_SIZE * 3)(%rdi) subq $-(VEC_SIZE * 4), %rdi cmpq %rdi, %rdx ja L(loop_4x_vec_forward) /* Store the last 4 * VEC. */ - VMOVU %VEC(5), (VEC_SIZE * 3)(%rdx) - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdx) - VMOVU %VEC(7), VEC_SIZE(%rdx) - VMOVU %VEC(8), (%rdx) + VMOVU %VMM(5), (VEC_SIZE * 3)(%rdx) + VMOVU %VMM(6), (VEC_SIZE * 2)(%rdx) + VMOVU %VMM(7), VEC_SIZE(%rdx) + VMOVU %VMM(8), (%rdx) /* Store the first VEC. */ - VMOVU %VEC(0), (%rcx) + VMOVU %VMM(0), (%rcx) /* Keep L(nop_backward) target close to jmp for 2-byte encoding. */ L(nop_backward): @@ -523,12 +515,12 @@ L(more_8x_vec_backward): addresses. */ /* First vec was also loaded into VEC(0). */ - VMOVU VEC_SIZE(%rsi), %VEC(5) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6) + VMOVU VEC_SIZE(%rsi), %VMM(5) + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(6) /* Begining of region for 4x backward copy stored in rcx. */ leaq (VEC_SIZE * -4 + -1)(%rdi, %rdx), %rcx - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7) - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(8) + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(7) + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(8) /* Subtract dst from src. Add back after dst aligned. */ subq %rdi, %rsi /* Align dst. */ @@ -540,25 +532,25 @@ L(more_8x_vec_backward): .p2align 4,, 11 L(loop_4x_vec_backward): /* Copy 4 * VEC a time backward. */ - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(1) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 1)(%rsi), %VEC(3) - VMOVU (VEC_SIZE * 0)(%rsi), %VEC(4) + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) + VMOVU (VEC_SIZE * 1)(%rsi), %VMM(3) + VMOVU (VEC_SIZE * 0)(%rsi), %VMM(4) addq $(VEC_SIZE * -4), %rsi - VMOVA %VEC(1), (VEC_SIZE * 3)(%rcx) - VMOVA %VEC(2), (VEC_SIZE * 2)(%rcx) - VMOVA %VEC(3), (VEC_SIZE * 1)(%rcx) - VMOVA %VEC(4), (VEC_SIZE * 0)(%rcx) + VMOVA %VMM(1), (VEC_SIZE * 3)(%rcx) + VMOVA %VMM(2), (VEC_SIZE * 2)(%rcx) + VMOVA %VMM(3), (VEC_SIZE * 1)(%rcx) + VMOVA %VMM(4), (VEC_SIZE * 0)(%rcx) addq $(VEC_SIZE * -4), %rcx cmpq %rcx, %rdi jb L(loop_4x_vec_backward) /* Store the first 4 * VEC. */ - VMOVU %VEC(0), (%rdi) - VMOVU %VEC(5), VEC_SIZE(%rdi) - VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi) - VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi) + VMOVU %VMM(0), (%rdi) + VMOVU %VMM(5), VEC_SIZE(%rdi) + VMOVU %VMM(6), (VEC_SIZE * 2)(%rdi) + VMOVU %VMM(7), (VEC_SIZE * 3)(%rdi) /* Store the last VEC. */ - VMOVU %VEC(8), -VEC_SIZE(%rdx, %rdi) + VMOVU %VMM(8), -VEC_SIZE(%rdx, %rdi) VZEROUPPER_RETURN #if defined USE_MULTIARCH && IS_IN (libc) @@ -568,7 +560,7 @@ L(loop_4x_vec_backward): # if ALIGN_MOVSB L(skip_short_movsb_check): # if MOVSB_ALIGN_TO > VEC_SIZE - VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU VEC_SIZE(%rsi), %VMM(1) # endif # if MOVSB_ALIGN_TO > (VEC_SIZE * 2) # error Unsupported MOVSB_ALIGN_TO @@ -597,9 +589,9 @@ L(skip_short_movsb_check): rep movsb - VMOVU %VEC(0), (%r8) + VMOVU %VMM(0), (%r8) # if MOVSB_ALIGN_TO > VEC_SIZE - VMOVU %VEC(1), VEC_SIZE(%r8) + VMOVU %VMM(1), VEC_SIZE(%r8) # endif VZEROUPPER_RETURN # endif @@ -640,7 +632,7 @@ L(movsb): # endif # if ALIGN_MOVSB # if MOVSB_ALIGN_TO > VEC_SIZE - VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU VEC_SIZE(%rsi), %VMM(1) # endif # if MOVSB_ALIGN_TO > (VEC_SIZE * 2) # error Unsupported MOVSB_ALIGN_TO @@ -664,9 +656,9 @@ L(movsb_align_dst): rep movsb /* Store VECs loaded for aligning. */ - VMOVU %VEC(0), (%r8) + VMOVU %VMM(0), (%r8) # if MOVSB_ALIGN_TO > VEC_SIZE - VMOVU %VEC(1), VEC_SIZE(%r8) + VMOVU %VMM(1), VEC_SIZE(%r8) # endif VZEROUPPER_RETURN # else /* !ALIGN_MOVSB. */ @@ -701,18 +693,18 @@ L(large_memcpy_2x): /* First vec was also loaded into VEC(0). */ # if VEC_SIZE < 64 - VMOVU VEC_SIZE(%rsi), %VEC(1) + VMOVU VEC_SIZE(%rsi), %VMM(1) # if VEC_SIZE < 32 - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) # endif # endif - VMOVU %VEC(0), (%rdi) + VMOVU %VMM(0), (%rdi) # if VEC_SIZE < 64 - VMOVU %VEC(1), VEC_SIZE(%rdi) + VMOVU %VMM(1), VEC_SIZE(%rdi) # if VEC_SIZE < 32 - VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi) - VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi) + VMOVU %VMM(2), (VEC_SIZE * 2)(%rdi) + VMOVU %VMM(3), (VEC_SIZE * 3)(%rdi) # endif # endif @@ -761,12 +753,12 @@ L(loop_large_memcpy_2x_inner): PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE) PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE + PREFETCHED_LOAD_SIZE * 2) /* Load vectors from rsi. */ - LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) + LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) + LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) subq $-LARGE_LOAD_SIZE, %rsi /* Non-temporal store vectors to rdi. */ - STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) - STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) + STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) + STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) subq $-LARGE_LOAD_SIZE, %rdi decl %ecx jnz L(loop_large_memcpy_2x_inner) @@ -785,31 +777,31 @@ L(loop_large_memcpy_2x_tail): /* Copy 4 * VEC a time forward with non-temporal stores. */ PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) - VMOVU (%rsi), %VEC(0) - VMOVU VEC_SIZE(%rsi), %VEC(1) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + VMOVU (%rsi), %VMM(0) + VMOVU VEC_SIZE(%rsi), %VMM(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) subq $-(VEC_SIZE * 4), %rsi addl $-(VEC_SIZE * 4), %edx - VMOVA %VEC(0), (%rdi) - VMOVA %VEC(1), VEC_SIZE(%rdi) - VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) - VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) + VMOVA %VMM(0), (%rdi) + VMOVA %VMM(1), VEC_SIZE(%rdi) + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi) + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi) subq $-(VEC_SIZE * 4), %rdi cmpl $(VEC_SIZE * 4), %edx ja L(loop_large_memcpy_2x_tail) L(large_memcpy_2x_end): /* Store the last 4 * VEC. */ - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) - - VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) - VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) - VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) - VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0) + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2) + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3) + + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx) + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx) + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx) + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx) VZEROUPPER_RETURN .p2align 4 @@ -831,16 +823,16 @@ L(loop_large_memcpy_4x_inner): PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 2 + PREFETCHED_LOAD_SIZE) PREFETCH_ONE_SET(1, (%rsi), PAGE_SIZE * 3 + PREFETCHED_LOAD_SIZE) /* Load vectors from rsi. */ - LOAD_ONE_SET((%rsi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) - LOAD_ONE_SET((%rsi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) - LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) - LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) + LOAD_ONE_SET((%rsi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) + LOAD_ONE_SET((%rsi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) + LOAD_ONE_SET((%rsi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11)) + LOAD_ONE_SET((%rsi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15)) subq $-LARGE_LOAD_SIZE, %rsi /* Non-temporal store vectors to rdi. */ - STORE_ONE_SET((%rdi), 0, %VEC(0), %VEC(1), %VEC(2), %VEC(3)) - STORE_ONE_SET((%rdi), PAGE_SIZE, %VEC(4), %VEC(5), %VEC(6), %VEC(7)) - STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VEC(8), %VEC(9), %VEC(10), %VEC(11)) - STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VEC(12), %VEC(13), %VEC(14), %VEC(15)) + STORE_ONE_SET((%rdi), 0, %VMM(0), %VMM(1), %VMM(2), %VMM(3)) + STORE_ONE_SET((%rdi), PAGE_SIZE, %VMM(4), %VMM(5), %VMM(6), %VMM(7)) + STORE_ONE_SET((%rdi), PAGE_SIZE * 2, %VMM(8), %VMM(9), %VMM(10), %VMM(11)) + STORE_ONE_SET((%rdi), PAGE_SIZE * 3, %VMM(12), %VMM(13), %VMM(14), %VMM(15)) subq $-LARGE_LOAD_SIZE, %rdi decl %ecx jnz L(loop_large_memcpy_4x_inner) @@ -858,31 +850,31 @@ L(loop_large_memcpy_4x_tail): /* Copy 4 * VEC a time forward with non-temporal stores. */ PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE) PREFETCH_ONE_SET (1, (%rdi), PREFETCHED_LOAD_SIZE) - VMOVU (%rsi), %VEC(0) - VMOVU VEC_SIZE(%rsi), %VEC(1) - VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2) - VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3) + VMOVU (%rsi), %VMM(0) + VMOVU VEC_SIZE(%rsi), %VMM(1) + VMOVU (VEC_SIZE * 2)(%rsi), %VMM(2) + VMOVU (VEC_SIZE * 3)(%rsi), %VMM(3) subq $-(VEC_SIZE * 4), %rsi addl $-(VEC_SIZE * 4), %edx - VMOVA %VEC(0), (%rdi) - VMOVA %VEC(1), VEC_SIZE(%rdi) - VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi) - VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi) + VMOVA %VMM(0), (%rdi) + VMOVA %VMM(1), VEC_SIZE(%rdi) + VMOVA %VMM(2), (VEC_SIZE * 2)(%rdi) + VMOVA %VMM(3), (VEC_SIZE * 3)(%rdi) subq $-(VEC_SIZE * 4), %rdi cmpl $(VEC_SIZE * 4), %edx ja L(loop_large_memcpy_4x_tail) L(large_memcpy_4x_end): /* Store the last 4 * VEC. */ - VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(0) - VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(1) - VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(2) - VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(3) - - VMOVU %VEC(0), -(VEC_SIZE * 4)(%rdi, %rdx) - VMOVU %VEC(1), -(VEC_SIZE * 3)(%rdi, %rdx) - VMOVU %VEC(2), -(VEC_SIZE * 2)(%rdi, %rdx) - VMOVU %VEC(3), -VEC_SIZE(%rdi, %rdx) + VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VMM(0) + VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VMM(1) + VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VMM(2) + VMOVU -VEC_SIZE(%rsi, %rdx), %VMM(3) + + VMOVU %VMM(0), -(VEC_SIZE * 4)(%rdi, %rdx) + VMOVU %VMM(1), -(VEC_SIZE * 3)(%rdi, %rdx) + VMOVU %VMM(2), -(VEC_SIZE * 2)(%rdi, %rdx) + VMOVU %VMM(3), -VEC_SIZE(%rdi, %rdx) VZEROUPPER_RETURN #endif END (MEMMOVE_SYMBOL (__memmove, unaligned_erms))