diff mbox series

[v4,5/5] x86: Double size of ERMS rep_movsb_threshold in dl-cacheinfo.h

Message ID 20211106183322.3129442-5-goldstein.w.n@gmail.com
State New
Headers show
Series [v4,1/5] string: Make tests birdirectional test-memcpy.c | expand

Commit Message

Noah Goldstein Nov. 6, 2021, 6:33 p.m. UTC 
No bug.

This patch doubles the rep_movsb_threshold when using ERMS. Based on
benchmarks the vector copy loop, especially now that it handles 4k
aliasing, is better for these medium ranged.

On Skylake with ERMS:

Size,   Align1, Align2, dst>src,(rep movsb) / (vec copy)
4096,   0,      0,      0,      0.975
4096,   0,      0,      1,      0.953
4096,   12,     0,      0,      0.969
4096,   12,     0,      1,      0.872
4096,   44,     0,      0,      0.979
4096,   44,     0,      1,      0.83
4096,   0,      12,     0,      1.006
4096,   0,      12,     1,      0.989
4096,   0,      44,     0,      0.739
4096,   0,      44,     1,      0.942
4096,   12,     12,     0,      1.009
4096,   12,     12,     1,      0.973
4096,   44,     44,     0,      0.791
4096,   44,     44,     1,      0.961
4096,   2048,   0,      0,      0.978
4096,   2048,   0,      1,      0.951
4096,   2060,   0,      0,      0.986
4096,   2060,   0,      1,      0.963
4096,   2048,   12,     0,      0.971
4096,   2048,   12,     1,      0.941
4096,   2060,   12,     0,      0.977
4096,   2060,   12,     1,      0.949
8192,   0,      0,      0,      0.85
8192,   0,      0,      1,      0.845
8192,   13,     0,      0,      0.937
8192,   13,     0,      1,      0.939
8192,   45,     0,      0,      0.932
8192,   45,     0,      1,      0.927
8192,   0,      13,     0,      0.621
8192,   0,      13,     1,      0.62
8192,   0,      45,     0,      0.53
8192,   0,      45,     1,      0.516
8192,   13,     13,     0,      0.664
8192,   13,     13,     1,      0.659
8192,   45,     45,     0,      0.593
8192,   45,     45,     1,      0.575
8192,   2048,   0,      0,      0.854
8192,   2048,   0,      1,      0.834
8192,   2061,   0,      0,      0.863
8192,   2061,   0,      1,      0.857
8192,   2048,   13,     0,      0.63
8192,   2048,   13,     1,      0.629
8192,   2061,   13,     0,      0.627
8192,   2061,   13,     1,      0.62
---
 sysdeps/x86/dl-cacheinfo.h   |  8 +++++---
 sysdeps/x86/dl-tunables.list | 26 +++++++++++++++-----------
 2 files changed, 20 insertions(+), 14 deletions(-)

Comments

H.J. Lu Nov. 6, 2021, 7:10 p.m. UTC | #1 
On Sat, Nov 6, 2021 at 11:36 AM Noah Goldstein via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>
> No bug.
>
> This patch doubles the rep_movsb_threshold when using ERMS. Based on
> benchmarks the vector copy loop, especially now that it handles 4k
> aliasing, is better for these medium ranged.
>
> On Skylake with ERMS:
>
> Size,   Align1, Align2, dst>src,(rep movsb) / (vec copy)
> 4096,   0,      0,      0,      0.975
> 4096,   0,      0,      1,      0.953
> 4096,   12,     0,      0,      0.969
> 4096,   12,     0,      1,      0.872
> 4096,   44,     0,      0,      0.979
> 4096,   44,     0,      1,      0.83
> 4096,   0,      12,     0,      1.006
> 4096,   0,      12,     1,      0.989
> 4096,   0,      44,     0,      0.739
> 4096,   0,      44,     1,      0.942
> 4096,   12,     12,     0,      1.009
> 4096,   12,     12,     1,      0.973
> 4096,   44,     44,     0,      0.791
> 4096,   44,     44,     1,      0.961
> 4096,   2048,   0,      0,      0.978
> 4096,   2048,   0,      1,      0.951
> 4096,   2060,   0,      0,      0.986
> 4096,   2060,   0,      1,      0.963
> 4096,   2048,   12,     0,      0.971
> 4096,   2048,   12,     1,      0.941
> 4096,   2060,   12,     0,      0.977
> 4096,   2060,   12,     1,      0.949
> 8192,   0,      0,      0,      0.85
> 8192,   0,      0,      1,      0.845
> 8192,   13,     0,      0,      0.937
> 8192,   13,     0,      1,      0.939
> 8192,   45,     0,      0,      0.932
> 8192,   45,     0,      1,      0.927
> 8192,   0,      13,     0,      0.621
> 8192,   0,      13,     1,      0.62
> 8192,   0,      45,     0,      0.53
> 8192,   0,      45,     1,      0.516
> 8192,   13,     13,     0,      0.664
> 8192,   13,     13,     1,      0.659
> 8192,   45,     45,     0,      0.593
> 8192,   45,     45,     1,      0.575
> 8192,   2048,   0,      0,      0.854
> 8192,   2048,   0,      1,      0.834
> 8192,   2061,   0,      0,      0.863
> 8192,   2061,   0,      1,      0.857
> 8192,   2048,   13,     0,      0.63
> 8192,   2048,   13,     1,      0.629
> 8192,   2061,   13,     0,      0.627
> 8192,   2061,   13,     1,      0.62
> ---
>  sysdeps/x86/dl-cacheinfo.h   |  8 +++++---
>  sysdeps/x86/dl-tunables.list | 26 +++++++++++++++-----------
>  2 files changed, 20 insertions(+), 14 deletions(-)
>
> diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
> index e6c94dfd02..2e43e67e4f 100644
> --- a/sysdeps/x86/dl-cacheinfo.h
> +++ b/sysdeps/x86/dl-cacheinfo.h
> @@ -866,12 +866,14 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
>    /* NB: The REP MOVSB threshold must be greater than VEC_SIZE * 8.  */
>    unsigned int minimum_rep_movsb_threshold;
>  #endif
> -  /* NB: The default REP MOVSB threshold is 2048 * (VEC_SIZE / 16).  */
> +  /* NB: The default REP MOVSB threshold is 4096 * (VEC_SIZE / 16) for
> +     VEC_SIZE == 64 or 32.  For VEC_SIZE == 16, the default REP MOVSB
> +     threshold is 2048 * (VEC_SIZE / 16).  */
>    unsigned int rep_movsb_threshold;
>    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
>        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
>      {
> -      rep_movsb_threshold = 2048 * (64 / 16);
> +      rep_movsb_threshold = 4096 * (64 / 16);
>  #if HAVE_TUNABLES
>        minimum_rep_movsb_threshold = 64 * 8;
>  #endif
> @@ -879,7 +881,7 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
>    else if (CPU_FEATURE_PREFERRED_P (cpu_features,
>                                     AVX_Fast_Unaligned_Load))
>      {
> -      rep_movsb_threshold = 2048 * (32 / 16);
> +      rep_movsb_threshold = 4096 * (32 / 16);
>  #if HAVE_TUNABLES
>        minimum_rep_movsb_threshold = 32 * 8;
>  #endif
> diff --git a/sysdeps/x86/dl-tunables.list b/sysdeps/x86/dl-tunables.list
> index dd6e1d65c9..419313804d 100644
> --- a/sysdeps/x86/dl-tunables.list
> +++ b/sysdeps/x86/dl-tunables.list
> @@ -32,17 +32,21 @@ glibc {
>      }
>      x86_rep_movsb_threshold {
>        type: SIZE_T
> -      # Since there is overhead to set up REP MOVSB operation, REP MOVSB
> -      # isn't faster on short data.  The memcpy micro benchmark in glibc
> -      # shows that 2KB is the approximate value above which REP MOVSB
> -      # becomes faster than SSE2 optimization on processors with Enhanced
> -      # REP MOVSB.  Since larger register size can move more data with a
> -      # single load and store, the threshold is higher with larger register
> -      # size.  Note: Since the REP MOVSB threshold must be greater than 8
> -      # times of vector size and the default value is 2048 * (vector size
> -      # / 16), the default value and the minimum value must be updated at
> -      # run-time.  NB: Don't set the default value since we can't tell if
> -      # the tunable value is set by user or not [BZ #27069].
> +      # Since there is overhead to set up REP MOVSB operation, REP
> +      # MOVSB isn't faster on short data.  The memcpy micro benchmark
> +      # in glibc shows that 2KB is the approximate value above which
> +      # REP MOVSB becomes faster than SSE2 optimization on processors
> +      # with Enhanced REP MOVSB.  Since larger register size can move
> +      # more data with a single load and store, the threshold is
> +      # higher with larger register size.  Micro benchmarks show AVX
> +      # REP MOVSB becomes faster apprximately at 8KB.  The AVX512
> +      # threshold is extrapolated to 16KB.  For machines with FSRM the
> +      # threshold is universally set at 2112 bytes.  Note: Since the
> +      # REP MOVSB threshold must be greater than 8 times of vector
> +      # size and the default value is 4096 * (vector size / 16), the
> +      # default value and the minimum value must be updated at
> +      # run-time.  NB: Don't set the default value since we can't tell
> +      # if the tunable value is set by user or not [BZ #27069].
>        minval: 1
>      }
>      x86_rep_stosb_threshold {
> --
> 2.25.1
>

LGTM.

Reviewed-by: H.J. Lu <hjl.tools@gmail.com>

Thanks.
Sunil Pandey April 23, 2022, 1:42 a.m. UTC | #2 
On Sat, Nov 6, 2021 at 12:11 PM H.J. Lu via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>
> On Sat, Nov 6, 2021 at 11:36 AM Noah Goldstein via Libc-alpha
> <libc-alpha@sourceware.org> wrote:
> >
> > No bug.
> >
> > This patch doubles the rep_movsb_threshold when using ERMS. Based on
> > benchmarks the vector copy loop, especially now that it handles 4k
> > aliasing, is better for these medium ranged.
> >
> > On Skylake with ERMS:
> >
> > Size,   Align1, Align2, dst>src,(rep movsb) / (vec copy)
> > 4096,   0,      0,      0,      0.975
> > 4096,   0,      0,      1,      0.953
> > 4096,   12,     0,      0,      0.969
> > 4096,   12,     0,      1,      0.872
> > 4096,   44,     0,      0,      0.979
> > 4096,   44,     0,      1,      0.83
> > 4096,   0,      12,     0,      1.006
> > 4096,   0,      12,     1,      0.989
> > 4096,   0,      44,     0,      0.739
> > 4096,   0,      44,     1,      0.942
> > 4096,   12,     12,     0,      1.009
> > 4096,   12,     12,     1,      0.973
> > 4096,   44,     44,     0,      0.791
> > 4096,   44,     44,     1,      0.961
> > 4096,   2048,   0,      0,      0.978
> > 4096,   2048,   0,      1,      0.951
> > 4096,   2060,   0,      0,      0.986
> > 4096,   2060,   0,      1,      0.963
> > 4096,   2048,   12,     0,      0.971
> > 4096,   2048,   12,     1,      0.941
> > 4096,   2060,   12,     0,      0.977
> > 4096,   2060,   12,     1,      0.949
> > 8192,   0,      0,      0,      0.85
> > 8192,   0,      0,      1,      0.845
> > 8192,   13,     0,      0,      0.937
> > 8192,   13,     0,      1,      0.939
> > 8192,   45,     0,      0,      0.932
> > 8192,   45,     0,      1,      0.927
> > 8192,   0,      13,     0,      0.621
> > 8192,   0,      13,     1,      0.62
> > 8192,   0,      45,     0,      0.53
> > 8192,   0,      45,     1,      0.516
> > 8192,   13,     13,     0,      0.664
> > 8192,   13,     13,     1,      0.659
> > 8192,   45,     45,     0,      0.593
> > 8192,   45,     45,     1,      0.575
> > 8192,   2048,   0,      0,      0.854
> > 8192,   2048,   0,      1,      0.834
> > 8192,   2061,   0,      0,      0.863
> > 8192,   2061,   0,      1,      0.857
> > 8192,   2048,   13,     0,      0.63
> > 8192,   2048,   13,     1,      0.629
> > 8192,   2061,   13,     0,      0.627
> > 8192,   2061,   13,     1,      0.62
> > ---
> >  sysdeps/x86/dl-cacheinfo.h   |  8 +++++---
> >  sysdeps/x86/dl-tunables.list | 26 +++++++++++++++-----------
> >  2 files changed, 20 insertions(+), 14 deletions(-)
> >
> > diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
> > index e6c94dfd02..2e43e67e4f 100644
> > --- a/sysdeps/x86/dl-cacheinfo.h
> > +++ b/sysdeps/x86/dl-cacheinfo.h
> > @@ -866,12 +866,14 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> >    /* NB: The REP MOVSB threshold must be greater than VEC_SIZE * 8.  */
> >    unsigned int minimum_rep_movsb_threshold;
> >  #endif
> > -  /* NB: The default REP MOVSB threshold is 2048 * (VEC_SIZE / 16).  */
> > +  /* NB: The default REP MOVSB threshold is 4096 * (VEC_SIZE / 16) for
> > +     VEC_SIZE == 64 or 32.  For VEC_SIZE == 16, the default REP MOVSB
> > +     threshold is 2048 * (VEC_SIZE / 16).  */
> >    unsigned int rep_movsb_threshold;
> >    if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
> >        && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
> >      {
> > -      rep_movsb_threshold = 2048 * (64 / 16);
> > +      rep_movsb_threshold = 4096 * (64 / 16);
> >  #if HAVE_TUNABLES
> >        minimum_rep_movsb_threshold = 64 * 8;
> >  #endif
> > @@ -879,7 +881,7 @@ dl_init_cacheinfo (struct cpu_features *cpu_features)
> >    else if (CPU_FEATURE_PREFERRED_P (cpu_features,
> >                                     AVX_Fast_Unaligned_Load))
> >      {
> > -      rep_movsb_threshold = 2048 * (32 / 16);
> > +      rep_movsb_threshold = 4096 * (32 / 16);
> >  #if HAVE_TUNABLES
> >        minimum_rep_movsb_threshold = 32 * 8;
> >  #endif
> > diff --git a/sysdeps/x86/dl-tunables.list b/sysdeps/x86/dl-tunables.list
> > index dd6e1d65c9..419313804d 100644
> > --- a/sysdeps/x86/dl-tunables.list
> > +++ b/sysdeps/x86/dl-tunables.list
> > @@ -32,17 +32,21 @@ glibc {
> >      }
> >      x86_rep_movsb_threshold {
> >        type: SIZE_T
> > -      # Since there is overhead to set up REP MOVSB operation, REP MOVSB
> > -      # isn't faster on short data.  The memcpy micro benchmark in glibc
> > -      # shows that 2KB is the approximate value above which REP MOVSB
> > -      # becomes faster than SSE2 optimization on processors with Enhanced
> > -      # REP MOVSB.  Since larger register size can move more data with a
> > -      # single load and store, the threshold is higher with larger register
> > -      # size.  Note: Since the REP MOVSB threshold must be greater than 8
> > -      # times of vector size and the default value is 2048 * (vector size
> > -      # / 16), the default value and the minimum value must be updated at
> > -      # run-time.  NB: Don't set the default value since we can't tell if
> > -      # the tunable value is set by user or not [BZ #27069].
> > +      # Since there is overhead to set up REP MOVSB operation, REP
> > +      # MOVSB isn't faster on short data.  The memcpy micro benchmark
> > +      # in glibc shows that 2KB is the approximate value above which
> > +      # REP MOVSB becomes faster than SSE2 optimization on processors
> > +      # with Enhanced REP MOVSB.  Since larger register size can move
> > +      # more data with a single load and store, the threshold is
> > +      # higher with larger register size.  Micro benchmarks show AVX
> > +      # REP MOVSB becomes faster apprximately at 8KB.  The AVX512
> > +      # threshold is extrapolated to 16KB.  For machines with FSRM the
> > +      # threshold is universally set at 2112 bytes.  Note: Since the
> > +      # REP MOVSB threshold must be greater than 8 times of vector
> > +      # size and the default value is 4096 * (vector size / 16), the
> > +      # default value and the minimum value must be updated at
> > +      # run-time.  NB: Don't set the default value since we can't tell
> > +      # if the tunable value is set by user or not [BZ #27069].
> >        minval: 1
> >      }
> >      x86_rep_stosb_threshold {
> > --
> > 2.25.1
> >
>
> LGTM.
>
> Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
>
> Thanks.
>
> --
> H.J.

I would like to backport this patch to release branches.
Any comments or objections?

--Sunil
diff mbox series

Patch

diff --git a/sysdeps/x86/dl-cacheinfo.h b/sysdeps/x86/dl-cacheinfo.h
index e6c94dfd02..2e43e67e4f 100644
--- a/sysdeps/x86/dl-cacheinfo.h
+++ b/sysdeps/x86/dl-cacheinfo.h
@@ -866,12 +866,14 @@  dl_init_cacheinfo (struct cpu_features *cpu_features)
   /* NB: The REP MOVSB threshold must be greater than VEC_SIZE * 8.  */
   unsigned int minimum_rep_movsb_threshold;
 #endif
-  /* NB: The default REP MOVSB threshold is 2048 * (VEC_SIZE / 16).  */
+  /* NB: The default REP MOVSB threshold is 4096 * (VEC_SIZE / 16) for
+     VEC_SIZE == 64 or 32.  For VEC_SIZE == 16, the default REP MOVSB
+     threshold is 2048 * (VEC_SIZE / 16).  */
   unsigned int rep_movsb_threshold;
   if (CPU_FEATURE_USABLE_P (cpu_features, AVX512F)
       && !CPU_FEATURE_PREFERRED_P (cpu_features, Prefer_No_AVX512))
     {
-      rep_movsb_threshold = 2048 * (64 / 16);
+      rep_movsb_threshold = 4096 * (64 / 16);
 #if HAVE_TUNABLES
       minimum_rep_movsb_threshold = 64 * 8;
 #endif
@@ -879,7 +881,7 @@  dl_init_cacheinfo (struct cpu_features *cpu_features)
   else if (CPU_FEATURE_PREFERRED_P (cpu_features,
 				    AVX_Fast_Unaligned_Load))
     {
-      rep_movsb_threshold = 2048 * (32 / 16);
+      rep_movsb_threshold = 4096 * (32 / 16);
 #if HAVE_TUNABLES
       minimum_rep_movsb_threshold = 32 * 8;
 #endif
diff --git a/sysdeps/x86/dl-tunables.list b/sysdeps/x86/dl-tunables.list
index dd6e1d65c9..419313804d 100644
--- a/sysdeps/x86/dl-tunables.list
+++ b/sysdeps/x86/dl-tunables.list
@@ -32,17 +32,21 @@  glibc {
     }
     x86_rep_movsb_threshold {
       type: SIZE_T
-      # Since there is overhead to set up REP MOVSB operation, REP MOVSB
-      # isn't faster on short data.  The memcpy micro benchmark in glibc
-      # shows that 2KB is the approximate value above which REP MOVSB
-      # becomes faster than SSE2 optimization on processors with Enhanced
-      # REP MOVSB.  Since larger register size can move more data with a
-      # single load and store, the threshold is higher with larger register
-      # size.  Note: Since the REP MOVSB threshold must be greater than 8
-      # times of vector size and the default value is 2048 * (vector size
-      # / 16), the default value and the minimum value must be updated at
-      # run-time.  NB: Don't set the default value since we can't tell if
-      # the tunable value is set by user or not [BZ #27069].
+      # Since there is overhead to set up REP MOVSB operation, REP
+      # MOVSB isn't faster on short data.  The memcpy micro benchmark
+      # in glibc shows that 2KB is the approximate value above which
+      # REP MOVSB becomes faster than SSE2 optimization on processors
+      # with Enhanced REP MOVSB.  Since larger register size can move
+      # more data with a single load and store, the threshold is
+      # higher with larger register size.  Micro benchmarks show AVX
+      # REP MOVSB becomes faster apprximately at 8KB.  The AVX512
+      # threshold is extrapolated to 16KB.  For machines with FSRM the
+      # threshold is universally set at 2112 bytes.  Note: Since the
+      # REP MOVSB threshold must be greater than 8 times of vector
+      # size and the default value is 4096 * (vector size / 16), the
+      # default value and the minimum value must be updated at
+      # run-time.  NB: Don't set the default value since we can't tell
+      # if the tunable value is set by user or not [BZ #27069].
       minval: 1
     }
     x86_rep_stosb_threshold {