RISC-V: add RVV support for memrchr using IFUNC

From: daichengrong <daichengrong@iscas.ac.cn>

This patch introduces a check for assembler compilation support for RVV
This patch introduces RVV memrchr with IFUNC.
The implementation selects the RVV memrchr via dl_hwcap.

On BPI_F3, running the  bench-memrchr in glibc benchtests, it gets an average improvement of 114%.
On K230, the average speedup was 99%.

---
 config.h.in                                   |  3 +
 sysdeps/riscv/configure                       | 35 +++++++
 sysdeps/riscv/configure.ac                    | 25 +++++
 sysdeps/riscv/multiarch/memrchr_generic.c     | 35 +++++++
 sysdeps/riscv/multiarch/memrchr_rvv.S         | 96 +++++++++++++++++++
 .../unix/sysv/linux/riscv/multiarch/Makefile  |  8 ++
 .../linux/riscv/multiarch/ifunc-impl-list.c   | 17 ++++
 .../unix/sysv/linux/riscv/multiarch/memrchr.c | 70 ++++++++++++++
 8 files changed, 289 insertions(+)
 create mode 100644 sysdeps/riscv/multiarch/memrchr_generic.c
 create mode 100644 sysdeps/riscv/multiarch/memrchr_rvv.S
 create mode 100644 sysdeps/unix/sysv/linux/riscv/multiarch/memrchr.c

On 13/03/25 05:43, daichengrong@iscas.ac.cn wrote:
> From: daichengrong <daichengrong@iscas.ac.cn>
> 
> This patch introduces a check for assembler compilation support for RVV
> This patch introduces RVV memrchr with IFUNC.
> The implementation selects the RVV memrchr via dl_hwcap.
> 
> On BPI_F3, running the  bench-memrchr in glibc benchtests, it gets an average improvement of 114%.
> On K230, the average speedup was 99%.
> 

This patch failed CI [1]:

 FAIL: lint-makefiles 

It should be easy to fix, just add the new memrchr_generic/memrchr in
the correct ordem on Makefile rule.

[1] https://patchwork.sourceware.org/project/glibc/patch/20250313084305.3494161-1-daichengrong@iscas.ac.cn/

> ---
>  config.h.in                                   |  3 +
>  sysdeps/riscv/configure                       | 35 +++++++
>  sysdeps/riscv/configure.ac                    | 25 +++++
>  sysdeps/riscv/multiarch/memrchr_generic.c     | 35 +++++++
>  sysdeps/riscv/multiarch/memrchr_rvv.S         | 96 +++++++++++++++++++
>  .../unix/sysv/linux/riscv/multiarch/Makefile  |  8 ++
>  .../linux/riscv/multiarch/ifunc-impl-list.c   | 17 ++++
>  .../unix/sysv/linux/riscv/multiarch/memrchr.c | 70 ++++++++++++++
>  8 files changed, 289 insertions(+)
>  create mode 100644 sysdeps/riscv/multiarch/memrchr_generic.c
>  create mode 100644 sysdeps/riscv/multiarch/memrchr_rvv.S
>  create mode 100644 sysdeps/unix/sysv/linux/riscv/multiarch/memrchr.c
> 
> diff --git a/config.h.in b/config.h.in
> index cdbd555366..7802e8f9c4 100644
> --- a/config.h.in
> +++ b/config.h.in
> @@ -139,6 +139,9 @@
>  /* RISC-V floating-point ABI for ld.so.  */
>  #undef RISCV_ABI_FLEN
>  
> +/* Define if assembler supports vector instructions on RISC-V.  */
> +#undef HAVE_RISCV_ASM_VECTOR_SUPPORT
> +
>  /* LOONGARCH integer ABI for ld.so.  */
>  #undef LOONGARCH_ABI_GRLEN
>  
> diff --git a/sysdeps/riscv/configure b/sysdeps/riscv/configure
> index 3ae4ae3bdb..bbda6a0d4a 100644
> --- a/sysdeps/riscv/configure
> +++ b/sysdeps/riscv/configure
> @@ -83,3 +83,38 @@ if test "$libc_cv_static_pie_on_riscv" = yes; then
>  
>  fi
>  
> +# Check if assembler supports attribute riscv vector macro.
> +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gcc attribute riscv vector support" >&5
> +printf %s "checking for gcc attribute riscv vector support... " >&6; }
> +if test ${libc_cv_gcc_rvv+y}
> +then :
> +  printf %s "(cached) " >&6
> +else case e in #(
> +  e) cat > conftest.S <<EOF
> +foo:
> +  .option push
> +  .option arch, +v
> +  vsetivli t0, 8, e8, m8, ta, ma
> +  .option pop
> +  ret
> +EOF
> +libc_cv_gcc_rvv=no
> +if ${CC-asm} -c conftest.S -o conftest.o 1>&5 \
> +  2>&5 ; then
> +  libc_cv_gcc_rvv=yes
> +fi
> +rm -f conftest* ;;
> +esac
> +fi
> +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $libc_cv_gcc_rvv" >&5
> +printf "%s\n" "$libc_cv_gcc_rvv" >&6; }
> +
> +if test x"$libc_cv_gcc_rvv" = xyes; then
> +  printf "%s\n" "#define HAVE_RISCV_ASM_VECTOR_SUPPORT 1" >>confdefs.h
> +
> +fi
> +
> +config_vars="$config_vars
> +have-gcc-riscv-rvv = $libc_cv_gcc_rvv"
> +
> +
> diff --git a/sysdeps/riscv/configure.ac b/sysdeps/riscv/configure.ac
> index ee3d1ed014..27e0e51b1c 100644
> --- a/sysdeps/riscv/configure.ac
> +++ b/sysdeps/riscv/configure.ac
> @@ -43,3 +43,28 @@ EOF
>  if test "$libc_cv_static_pie_on_riscv" = yes; then
>    AC_DEFINE(SUPPORT_STATIC_PIE)
>  fi
> +
> +# Check if assembler supports attribute riscv vector macro.
> +AC_CACHE_CHECK([for gcc attribute riscv vector support],
> +        libc_cv_gcc_rvv, [dnl
> +cat > conftest.S <<EOF
> +foo:
> +  .option push
> +  .option arch, +v
> +  vsetivli t0, 8, e8, m8, ta, ma
> +  .option pop
> +  ret
> +EOF
> +libc_cv_gcc_rvv=no
> +if ${CC-asm} -c conftest.S -o conftest.o 1>&AS_MESSAGE_LOG_FD \
> +  2>&AS_MESSAGE_LOG_FD ; then
> +  libc_cv_gcc_rvv=yes
> +fi
> +rm -f conftest*])
> +
> +if test x"$libc_cv_gcc_rvv" = xyes; then
> +  AC_DEFINE(HAVE_RISCV_ASM_VECTOR_SUPPORT)
> +fi
> +
> +LIBC_CONFIG_VAR([have-gcc-riscv-rvv], [$libc_cv_gcc_rvv])
> +
> diff --git a/sysdeps/riscv/multiarch/memrchr_generic.c b/sysdeps/riscv/multiarch/memrchr_generic.c
> new file mode 100644
> index 0000000000..c0a146eb62
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/memrchr_generic.c
> @@ -0,0 +1,35 @@
> +/* Re-include the default memrchr implementation.
> +   Copyright (C) 2018-2025 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +#include <sysdep.h>
> +
> +#include <string.h>
> +#if IS_IN (libc)
> +
> +# define MEMRCHR __memrchr_generic
> +
> +/* Do not hide the generic version of memrchr, we use it internally.  */
> +# undef libc_hidden_builtin_def
> +# define libc_hidden_builtin_def(name)
> +
> +# undef weak_alias
> +# define weak_alias(a, b)
> +
> +#endif
> +
> +#include <string/memrchr.c>
> diff --git a/sysdeps/riscv/multiarch/memrchr_rvv.S b/sysdeps/riscv/multiarch/memrchr_rvv.S
> new file mode 100644
> index 0000000000..6fa801023f
> --- /dev/null
> +++ b/sysdeps/riscv/multiarch/memrchr_rvv.S
> @@ -0,0 +1,96 @@
> +/* Optimized memrchr implementation using RVV.
> +   Copyright (C) 2018-2025 Free Software Foundation, Inc.
> +
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library.  If not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +#include <sysdep.h>
> +
> +#define ELEM_LMUL_1_SETTING m1
> +#define ELEM_LMUL_8_SETTING m8
> +#define ELEM_SEW_8_SETTING e8
> +
> +#define v_seq_set_data v0
> +#define v_loaded_data v8
> +#define v_reverse_loaded_data v16
> +#define v_index v16
> +#define v_reverse_index v24
> +
> +#define srcin		a0
> +#define chrin		a1
> +#define cntin		a2
> +#define result		a0
> +
> +#define	tmp_end		t1
> +#define	loaded_data_start		t1
> +
> +#define VL a4
> +
> +#define cntrem		t2
> +#define set_num    t3
> +#define first_set_index    t4
> +#define	loaded_data_max_index		t5
> +#define	one		t6
> +
> +ENTRY (__memrchr_rvv)
> +.option push
> +.option arch, +v
> +
> +    mv cntrem, cntin
> +    add tmp_end, srcin, cntrem
> +    
> +L(memrchr_loop):
> +    blez cntrem, L(memrchr_nomatch)
> +    vsetvli VL, cntrem, ELEM_SEW_8_SETTING, ELEM_LMUL_8_SETTING, ta, ma
> +    sub loaded_data_start, tmp_end, VL
> +    vle8.v v_loaded_data, (loaded_data_start)
> +    sub cntrem, cntrem, VL
> +    /*  Set v0[i] where v8[i] = a1 */
> +    vmseq.vx v_seq_set_data, v_loaded_data, chrin 
> +    /*  count the number of equal elements  */  
> +    vcpop.m set_num, v_seq_set_data
> +    beqz set_num, L(memrchr_loop)
> +
> +L(memrchr_found):
> +    li one, 1
> +    bgt set_num, one, L(memrchr_multi_found)
> +    /* get the first equal element index */ 
> +    vfirst.m first_set_index, v_seq_set_data
> +    add result, loaded_data_start, first_set_index 
> +    ret
> +
> +L(memrchr_multi_found):
> +    /* index [0, 1, 2, 3, ...]*/
> +    vid.v v_index 
> +    addi loaded_data_max_index, VL, -1 
> +    /* index [VL-1, VL-2, ..., 0] */
> +    vrsub.vx v_reverse_index, v_index, loaded_data_max_index
> +    /* reverse loaded data */ 
> +    vrgather.vv v_reverse_loaded_data, v_loaded_data, v_reverse_index
> +    /*  Set v0[i] where v8[i] = a1 */
> +    vmseq.vx v_seq_set_data, v_reverse_loaded_data, chrin
> +    /* get the first equal element index of reverse data*/ 
> +    vfirst.m first_set_index, v_seq_set_data
> +    /* calc the true index of data*/ 
> +    sub first_set_index, loaded_data_max_index, first_set_index
> +    add result, loaded_data_start, first_set_index 
> +    ret
> +    
> +L(memrchr_nomatch):
> +	mv	result, zero
> +	ret    
> +.option pop
> +END (__memrchr_rvv)
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> index fcef5659d4..64b20d7074 100644
> --- a/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/Makefile
> @@ -1,9 +1,17 @@
>  ifeq ($(subdir),string)
>  sysdep_routines += \
> +  memrchr \
> +  memrchr_generic \
>    memcpy \
>    memcpy-generic \
>    memcpy_noalignment \
>    # sysdep_routines
>  
> +ifeq ($(have-gcc-riscv-rvv),yes)
> +sysdep_routines += \
> +  memrchr_rvv \
> +  # rvv sysdep_routines
> +endif
> +
>  CFLAGS-memcpy_noalignment.c += -mno-strict-align
>  endif
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> index 1c1deca8f6..deb787a116 100644
> --- a/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/ifunc-impl-list.c
> @@ -19,6 +19,8 @@
>  #include <ifunc-impl-list.h>
>  #include <string.h>
>  #include <sys/hwprobe.h>
> +#include <ldsodefs.h>
> +#include <asm/hwcap.h>
>  
>  size_t
>  __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
> @@ -27,6 +29,9 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>    size_t i = max;
>  
>    bool fast_unaligned = false;
> +#if defined(HAVE_RISCV_ASM_VECTOR_SUPPORT) 
> +  bool rvv_ext = false;
> +#endif
>  
>    struct riscv_hwprobe pair = { .key = RISCV_HWPROBE_KEY_CPUPERF_0 };
>    if (__riscv_hwprobe (&pair, 1, 0, NULL, 0) == 0
> @@ -34,6 +39,18 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
>            == RISCV_HWPROBE_MISALIGNED_FAST)
>      fast_unaligned = true;
>  
> +#if defined(HAVE_RISCV_ASM_VECTOR_SUPPORT) 
> +  if (GLRO(dl_hwcap) & COMPAT_HWCAP_ISA_V) 
> +    rvv_ext = true;
> +#endif
> +    
> +IFUNC_IMPL (i, name, memrchr,
> +  #if defined(HAVE_RISCV_ASM_VECTOR_SUPPORT) 
> +          IFUNC_IMPL_ADD (array, i, memrchr, rvv_ext,
> +            __memrchr_rvv)
> +  #endif
> +          IFUNC_IMPL_ADD (array, i, memrchr, 1, __memrchr_generic))
> +
>    IFUNC_IMPL (i, name, memcpy,
>  	      IFUNC_IMPL_ADD (array, i, memcpy, fast_unaligned,
>  			      __memcpy_noalignment)
> diff --git a/sysdeps/unix/sysv/linux/riscv/multiarch/memrchr.c b/sysdeps/unix/sysv/linux/riscv/multiarch/memrchr.c
> new file mode 100644
> index 0000000000..3f3a33deab
> --- /dev/null
> +++ b/sysdeps/unix/sysv/linux/riscv/multiarch/memrchr.c
> @@ -0,0 +1,70 @@
> +/* Multiple versions of memrchr. 
> +   Copyright (C) 2018-2025 Free Software Foundation, Inc.
> +   This file is part of the GNU C Library.
> +
> +   The GNU C Library is free software; you can redistribute it and/or
> +   modify it under the terms of the GNU Lesser General Public
> +   License as published by the Free Software Foundation; either
> +   version 2.1 of the License, or (at your option) any later version.
> +
> +   The GNU C Library is distributed in the hope that it will be useful,
> +   but WITHOUT ANY WARRANTY; without even the implied warranty of
> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> +   Lesser General Public License for more details.
> +
> +   You should have received a copy of the GNU Lesser General Public
> +   License along with the GNU C Library; if not, see
> +   <https://www.gnu.org/licenses/>.  */
> +
> +/* Define multiple versions only for the definition in libc.  */
> +
> +#if IS_IN (libc)
> +# undef memrchr
> +# define memrchr __redirect_memrchr
> +# define __memrchr __redirect___memrchr
> +
> +# include <stdint.h>
> +# include <string.h>
> +# include <ifunc-init.h>
> +# include <riscv-ifunc.h>
> +# include <sys/hwprobe.h>
> +# include <asm/hwcap.h>
> +
> +extern __typeof (__redirect_memrchr) ___memrchr;
> +
> +extern __typeof (__redirect_memrchr) __memrchr_generic attribute_hidden;
> +extern __typeof (__redirect_memrchr) __memrchr_rvv attribute_hidden;
> +static inline __typeof (__redirect_memrchr) *
> +select_memrchr_ifunc (uint64_t dl_hwcap, __riscv_hwprobe_t hwprobe_func)
> +{
> +
> +#if defined(HAVE_RISCV_ASM_VECTOR_SUPPORT) 
> +  if (dl_hwcap & COMPAT_HWCAP_ISA_V) 
> +  {
> +    return __memrchr_rvv;
> +  }
> +#endif
> +
> +
> +  return __memrchr_generic;
> +}
> +
> +riscv_libc_ifunc (___memrchr, select_memrchr_ifunc);
> +
> +
> +# undef memrchr
> +# undef __memrchr
> +strong_alias (___memrchr, memrchr);
> +strong_alias (___memrchr, __memrchr);
> +
> +# ifdef SHARED
> +__hidden_ver1 (memrchr, __GI_memrchr, __redirect_memrchr)
> +  __attribute__ ((visibility ("hidden"))) __attribute_copy__ (memrchr);
> +
> +__hidden_ver1 (memrchr, __GI___memrchr, __redirect___memrchr)
> +  __attribute__ ((visibility ("hidden"))) __attribute_copy__ (memrchr);
> +# endif
> +
> +#else
> +# include <string/memrchr.c>
> +#endif