@@ -2593,6 +2593,16 @@ check_simd_lane_bounds (location_t location, const aarch64_pragma_builtins_data
vector_to_index_mode_size / 4 - 1);
break;
+ case UNSPEC_FMLALB:
+ case UNSPEC_FMLALT:
+ case UNSPEC_FMLALLBB:
+ case UNSPEC_FMLALLBT:
+ case UNSPEC_FMLALLTB:
+ case UNSPEC_FMLALLTT:
+ require_immediate_range (location, index_arg, 0,
+ vector_to_index_mode_size - 1);
+ break;
+
default:
gcc_unreachable ();
}
@@ -263,6 +263,8 @@ aarch64_update_cpp_builtins (cpp_reader *pfile)
aarch64_def_or_undef (TARGET_FP8DOT4, "__ARM_FEATURE_FP8DOT4", pfile);
+ aarch64_def_or_undef (TARGET_FP8FMA, "__ARM_FEATURE_FP8FMA", pfile);
+
aarch64_def_or_undef (TARGET_LS64,
"__ARM_FEATURE_LS64", pfile);
aarch64_def_or_undef (TARGET_RCPC, "__ARM_FEATURE_RCPC", pfile);
@@ -238,6 +238,8 @@ AARCH64_OPT_EXTENSION("fp8dot2", FP8DOT2, (SIMD), (), (), "fp8dot2")
AARCH64_OPT_EXTENSION("fp8dot4", FP8DOT4, (SIMD), (), (), "fp8dot4")
+AARCH64_OPT_EXTENSION("fp8fma", FP8FMA, (SIMD), (), (), "fp8fma")
+
AARCH64_OPT_EXTENSION("faminmax", FAMINMAX, (SIMD), (), (), "faminmax")
#undef AARCH64_OPT_FMV_EXTENSION
@@ -48,6 +48,15 @@
ENTRY_TERNARY_FPM_LANE (vdotq_laneq_##T##_mf8_fpm, ternary_fpm_lane, T##q, \
T##q, f8q, f8q, U)
+#undef ENTRY_FMA_FPM
+#define ENTRY_FMA_FPM(N, T, U) \
+ ENTRY_TERNARY_FPM (N##_##T##_mf8_fpm, ternary_fpm, \
+ T##q, T##q, f8q, f8q, U) \
+ ENTRY_TERNARY_FPM_LANE (N##_lane_##T##_mf8_fpm, ternary_fpm_lane, \
+ T##q, T##q, f8q, f8, U) \
+ ENTRY_TERNARY_FPM_LANE (N##_laneq_##T##_mf8_fpm, ternary_fpm_lane, \
+ T##q, T##q, f8q, f8q, U)
+
#undef ENTRY_UNARY_FPM
#define ENTRY_UNARY_FPM(N, S, T0, T1, U) \
ENTRY (N, S, T0, T1, none, none, none, U)
@@ -121,3 +130,13 @@ ENTRY_VDOT_FPM (f16, UNSPEC_VDOT2)
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8DOT4)
ENTRY_VDOT_FPM (f32, UNSPEC_VDOT4)
#undef REQUIRED_EXTENSIONS
+
+// fp8 multiply-add
+#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8FMA)
+ENTRY_FMA_FPM (vmlalbq, f16, UNSPEC_FMLALB)
+ENTRY_FMA_FPM (vmlaltq, f16, UNSPEC_FMLALT)
+ENTRY_FMA_FPM (vmlallbbq, f32, UNSPEC_FMLALLBB)
+ENTRY_FMA_FPM (vmlallbtq, f32, UNSPEC_FMLALLBT)
+ENTRY_FMA_FPM (vmlalltbq, f32, UNSPEC_FMLALLTB)
+ENTRY_FMA_FPM (vmlallttq, f32, UNSPEC_FMLALLTT)
+#undef REQUIRED_EXTENSIONS
@@ -10125,3 +10125,32 @@
"TARGET_FP8DOT4"
"<fpm_uns_op>\t%1.<VDQSF:Vtype>, %2.<VB:Vtype>, %3.<VDQSF:Vdotlanetype>[%4]"
)
+
+;; fpm fma instructions.
+(define_insn
+ "@aarch64_<fpm_uns_op><VQ_HSF:mode><VQ_HSF:mode><V16QI_ONLY:mode><V16QI_ONLY:mode>"
+ [(set (match_operand:VQ_HSF 0 "register_operand" "=w")
+ (unspec:VQ_HSF
+ [(match_operand:VQ_HSF 1 "register_operand" "w")
+ (match_operand:V16QI_ONLY 2 "register_operand" "w")
+ (match_operand:V16QI_ONLY 3 "register_operand" "w")
+ (reg:DI FPM_REGNUM)]
+ FPM_FMA_UNS))]
+ "TARGET_FP8FMA"
+ "<fpm_uns_op>\t%1.<VQ_HSF:Vtype>, %2.<V16QI_ONLY:Vtype>, %3.<V16QI_ONLY:Vtype>"
+)
+
+;; fpm fma instructions with lane.
+(define_insn
+ "@aarch64_<fpm_uns_op><VQ_HSF:mode><VQ_HSF:mode><V16QI_ONLY:mode><VB:mode><SI_ONLY:mode>"
+ [(set (match_operand:VQ_HSF 0 "register_operand" "=w")
+ (unspec:VQ_HSF
+ [(match_operand:VQ_HSF 1 "register_operand" "w")
+ (match_operand:V16QI_ONLY 2 "register_operand" "w")
+ (match_operand:VB 3 "register_operand" "w")
+ (match_operand:SI_ONLY 4 "const_int_operand" "n")
+ (reg:DI FPM_REGNUM)]
+ FPM_FMA_UNS))]
+ "TARGET_FP8FMA"
+ "<fpm_uns_op>\t%1.<VQ_HSF:Vtype>, %2.<V16QI_ONLY:Vtype>, %3.b[%4]"
+)
@@ -489,6 +489,9 @@ constexpr auto AARCH64_FL_DEFAULT_ISA_MODE ATTRIBUTE_UNUSED
/* fp8 dot product instructions are enabled through +fp8dot4. */
#define TARGET_FP8DOT4 AARCH64_HAVE_ISA (FP8DOT4)
+/* fp8 multiply-add instructions are enabled through +fp8fma. */
+#define TARGET_FP8FMA AARCH64_HAVE_ISA (FP8FMA)
+
/* Standard register usage. */
/* 31 64-bit general purpose registers R0-R30:
@@ -716,6 +716,10 @@
UNSPEC_FMINNMV ; Used in aarch64-simd.md.
UNSPEC_FMINV ; Used in aarch64-simd.md.
UNSPEC_FADDV ; Used in aarch64-simd.md.
+ UNSPEC_FMLALLBB ; Used in aarch64-simd.md.
+ UNSPEC_FMLALLBT ; Used in aarch64-simd.md.
+ UNSPEC_FMLALLTB ; Used in aarch64-simd.md.
+ UNSPEC_FMLALLTT ; Used in aarch64-simd.md.
UNSPEC_FNEG ; Used in aarch64-simd.md.
UNSPEC_FSCALE ; Used in aarch64-simd.md.
UNSPEC_ADDV ; Used in aarch64-simd.md.
@@ -4613,8 +4617,22 @@
(define_int_iterator FPM_VDOT4_UNS [UNSPEC_VDOT4])
+(define_int_iterator FPM_FMA_UNS
+ [UNSPEC_FMLALB
+ UNSPEC_FMLALT
+ UNSPEC_FMLALLBB
+ UNSPEC_FMLALLBT
+ UNSPEC_FMLALLTB
+ UNSPEC_FMLALLTT])
+
(define_int_attr fpm_uns_op
[(UNSPEC_FSCALE "fscale")
+ (UNSPEC_FMLALB "fmlalb")
+ (UNSPEC_FMLALT "fmlalt")
+ (UNSPEC_FMLALLBB "fmlallbb")
+ (UNSPEC_FMLALLBT "fmlallbt")
+ (UNSPEC_FMLALLTB "fmlalltb")
+ (UNSPEC_FMLALLTT "fmlalltt")
(UNSPEC_VCVT_F16 "fcvtn")
(UNSPEC_VCVTQ_F16 "fcvtn")
(UNSPEC_VCVT_F32 "fcvtn")
@@ -21809,6 +21809,8 @@ Enable the fp8 (8-bit floating point) extension.
Enable the fp8dot2 (8-bit floating point dot product) extension.
@item fp8dot4
Enable the fp8dot4 (8-bit floating point dot product) extension.
+@item fp8fma
+Enable the fp8fma (8-bit floating point multiply-add) extension.
@item faminmax
Enable the Floating Point Absolute Maximum/Minimum extension.
new file mode 100644
@@ -0,0 +1,221 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-O3 -march=armv9-a+fp8fma" } */
+/* { dg-final { check-function-bodies "**" "" } } */
+
+#include "arm_neon.h"
+
+/*
+** test_vmlalbq_f16_fpm:
+** msr fpmr, x0
+** fmlalb v0.8h, v1.16b, v2.16b
+** ret
+*/
+float16x8_t
+test_vmlalbq_f16_fpm (float16x8_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlalbq_f16_mf8_fpm (a, b, c, d);
+}
+
+/*
+** test_vmlaltq_f16_fpm:
+** msr fpmr, x0
+** fmlalt v0.8h, v1.16b, v2.16b
+** ret
+*/
+float16x8_t
+test_vmlaltq_f16_fpm (float16x8_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlaltq_f16_mf8_fpm (a, b, c, d);
+}
+
+/*
+** test_vmlallbbq_f32_fpm:
+** msr fpmr, x0
+** fmlallbb v0.4s, v1.16b, v2.16b
+** ret
+*/
+float32x4_t
+test_vmlallbbq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlallbbq_f32_mf8_fpm (a, b, c, d);
+}
+
+/*
+** test_vmlallbtq_f32_fpm:
+** msr fpmr, x0
+** fmlallbt v0.4s, v1.16b, v2.16b
+** ret
+*/
+float32x4_t
+test_vmlallbtq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlallbtq_f32_mf8_fpm (a, b, c, d);
+}
+
+/*
+** test_vmlalltbq_f32_fpm:
+** msr fpmr, x0
+** fmlalltb v0.4s, v1.16b, v2.16b
+** ret
+*/
+float32x4_t
+test_vmlalltbq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlalltbq_f32_mf8_fpm (a, b, c, d);
+}
+
+/*
+** test_vmlallttq_f32_fpm:
+** msr fpmr, x0
+** fmlalltt v0.4s, v1.16b, v2.16b
+** ret
+*/
+float32x4_t
+test_vmlallttq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlallttq_f32_mf8_fpm (a, b, c, d);
+}
+
+/*
+** test_vmlalbq_lane_f16_fpm:
+** msr fpmr, x0
+** fmlalb v0.8h, v1.16b, v2.b\[1\]
+** ret
+*/
+float16x8_t
+test_vmlalbq_lane_f16_fpm (float16x8_t a, mfloat8x16_t b, mfloat8x8_t c, fpm_t d)
+{
+ return vmlalbq_lane_f16_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlalbq_laneq_f16_fpm:
+** msr fpmr, x0
+** fmlalb v0.8h, v1.16b, v2.b\[1\]
+** ret
+*/
+float16x8_t
+test_vmlalbq_laneq_f16_fpm (float16x8_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlalbq_laneq_f16_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlaltq_lane_f16_fpm:
+** msr fpmr, x0
+** fmlalt v0.8h, v1.16b, v2.b\[1\]
+** ret
+*/
+float16x8_t
+test_vmlaltq_lane_f16_fpm (float16x8_t a, mfloat8x16_t b, mfloat8x8_t c, fpm_t d)
+{
+ return vmlaltq_lane_f16_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlaltq_laneq_f16_fpm:
+** msr fpmr, x0
+** fmlalt v0.8h, v1.16b, v2.b\[1\]
+** ret
+*/
+float16x8_t
+test_vmlaltq_laneq_f16_fpm (float16x8_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlaltq_laneq_f16_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlallbbq_lane_f32_fpm:
+** msr fpmr, x0
+** fmlallbb v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlallbbq_lane_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x8_t c, fpm_t d)
+{
+ return vmlallbbq_lane_f32_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlallbbq_laneq_f32_fpm:
+** msr fpmr, x0
+** fmlallbb v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlallbbq_laneq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlallbbq_laneq_f32_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlallbtq_lane_f32_fpm:
+** msr fpmr, x0
+** fmlallbt v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlallbtq_lane_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x8_t c, fpm_t d)
+{
+ return vmlallbtq_lane_f32_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlallbtq_laneq_f32_fpm:
+** msr fpmr, x0
+** fmlallbt v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlallbtq_laneq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlallbtq_laneq_f32_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlalltbq_lane_f32_fpm:
+** msr fpmr, x0
+** fmlalltb v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlalltbq_lane_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x8_t c, fpm_t d)
+{
+ return vmlalltbq_lane_f32_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlalltbq_laneq_f32_fpm:
+** msr fpmr, x0
+** fmlalltb v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlalltbq_laneq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlalltbq_laneq_f32_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlallttq_lane_f32_fpm:
+** msr fpmr, x0
+** fmlalltt v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlallttq_lane_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x8_t c, fpm_t d)
+{
+ return vmlallttq_lane_f32_mf8_fpm (a, b, c, 1, d);
+}
+
+/*
+** test_vmlallttq_laneq_f32_fpm:
+** msr fpmr, x0
+** fmlalltt v0.4s, v1.16b, v2.b\[1\]
+** ret
+*/
+float32x4_t
+test_vmlallttq_laneq_f32_fpm (float32x4_t a, mfloat8x16_t b, mfloat8x16_t c, fpm_t d)
+{
+ return vmlallttq_laneq_f32_mf8_fpm (a, b, c, 1, d);
+}
The AArch64 FEAT_FP8FMA extension introduces instructions for multiply-add of vectors. This patch introduces the following instructions: 1. {vmlalbq|vmlaltq}_f16_mf8_fpm. 2. {vmlalbq|vmlaltq}_lane{q}_f16_mf8_fpm. 3. {vmlallbbq|vmlallbtq|vmlalltbq|vmlallttq}_f32_mf8_fpm. 4. {vmlallbbq|vmlallbtq|vmlalltbq|vmlallttq}_lane{q}_f32_mf8_fpm. It introduces the fp8fma flag. gcc/ChangeLog: * config/aarch64/aarch64-builtins.cc (check_simd_lane_bounds): Add support for new unspecs. * config/aarch64/aarch64-c.cc (aarch64_update_cpp_builtins): New flags. * config/aarch64/aarch64-option-extensions.def (AARCH64_OPT_EXTENSION): New flags. * config/aarch64/aarch64-simd-pragma-builtins.def (ENTRY_FMA_FPM): Macro to declare fma intrinsics. * config/aarch64/aarch64-simd.md: New instruction pattern for fp8fma instructions. * config/aarch64/aarch64.h (TARGET_FP8FMA): New flag for fp8fma instructions. * config/aarch64/iterators.md: New attributes and iterators. * doc/invoke.texi: New flag for fp8fma instructions. gcc/testsuite/ChangeLog: * gcc.target/aarch64/simd/fma_fpm.c: New test. --- In the instruction pattern for handling lanes, I am not doing any casting of the the third operand and hardcoding the '.b' suffix in the assembly string. Is that okay? --- gcc/config/aarch64/aarch64-builtins.cc | 10 + gcc/config/aarch64/aarch64-c.cc | 2 + .../aarch64/aarch64-option-extensions.def | 2 + .../aarch64/aarch64-simd-pragma-builtins.def | 19 ++ gcc/config/aarch64/aarch64-simd.md | 29 +++ gcc/config/aarch64/aarch64.h | 3 + gcc/config/aarch64/iterators.md | 18 ++ gcc/doc/invoke.texi | 2 + .../gcc.target/aarch64/simd/fma_fpm.c | 221 ++++++++++++++++++ 9 files changed, 306 insertions(+) create mode 100644 gcc/testsuite/gcc.target/aarch64/simd/fma_fpm.c