@@ -624,4 +624,21 @@ poly_uint64 aarch64_regmode_natural_size (machine_mode);
bool aarch64_high_bits_all_ones_p (HOST_WIDE_INT);
+struct atomic_ool_names
+{
+ const char *str[4][4];
+};
+
+rtx aarch64_atomic_ool_func(machine_mode mode, rtx model_rtx,
+ const atomic_ool_names *names);
+extern const atomic_ool_names aarch64_ool_swp_names;
+extern const atomic_ool_names aarch64_ool_stadd_names;
+extern const atomic_ool_names aarch64_ool_stset_names;
+extern const atomic_ool_names aarch64_ool_stclr_names;
+extern const atomic_ool_names aarch64_ool_steor_names;
+extern const atomic_ool_names aarch64_ool_ldadd_names;
+extern const atomic_ool_names aarch64_ool_ldset_names;
+extern const atomic_ool_names aarch64_ool_ldclr_names;
+extern const atomic_ool_names aarch64_ool_ldeor_names;
+
#endif /* GCC_AARCH64_PROTOS_H */
@@ -14160,6 +14160,90 @@ aarch64_emit_unlikely_jump (rtx insn)
add_reg_br_prob_note (jump, profile_probability::very_unlikely ());
}
+/* We store the names of the various atomic helpers in a 4x4 array.
+ Return the libcall function given MODE, MODEL and NAMES. */
+
+rtx
+aarch64_atomic_ool_func(machine_mode mode, rtx model_rtx,
+ const atomic_ool_names *names)
+{
+ memmodel model = memmodel_base (INTVAL (model_rtx));
+ int mode_idx, model_idx;
+
+ switch (mode)
+ {
+ case E_QImode:
+ mode_idx = 0;
+ break;
+ case E_HImode:
+ mode_idx = 1;
+ break;
+ case E_SImode:
+ mode_idx = 2;
+ break;
+ case E_DImode:
+ mode_idx = 3;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ switch (model)
+ {
+ case MEMMODEL_RELAXED:
+ model_idx = 0;
+ break;
+ case MEMMODEL_CONSUME:
+ case MEMMODEL_ACQUIRE:
+ model_idx = 1;
+ break;
+ case MEMMODEL_RELEASE:
+ model_idx = 2;
+ break;
+ case MEMMODEL_ACQ_REL:
+ case MEMMODEL_SEQ_CST:
+ model_idx = 3;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ return init_one_libfunc_visibility (names->str[mode_idx][model_idx],
+ VISIBILITY_HIDDEN);
+}
+
+#define DEF0(B, N) \
+ { "__aa64_" #B #N "_relax", \
+ "__aa64_" #B #N "_acq", \
+ "__aa64_" #B #N "_rel", \
+ "__aa64_" #B #N "_acq_rel" }
+
+#define DEF4(B) DEF0(B, 1), DEF0(B, 2), DEF0(B, 4), DEF0(B, 8)
+
+static const atomic_ool_names aarch64_ool_cas_names = { { DEF4(cas) } };
+const atomic_ool_names aarch64_ool_swp_names = { { DEF4(swp) } };
+const atomic_ool_names aarch64_ool_ldadd_names = { { DEF4(ldadd) } };
+const atomic_ool_names aarch64_ool_ldset_names = { { DEF4(ldset) } };
+const atomic_ool_names aarch64_ool_ldclr_names = { { DEF4(ldclr) } };
+const atomic_ool_names aarch64_ool_ldeor_names = { { DEF4(ldeor) } };
+
+/* Note that the store forms are only available for relax and release
+ memory models. For the other models, re-use the load forms. */
+#undef DEF0
+#define DEF0(B, N) \
+ { "__aa64_st" #B #N "_relax", \
+ "__aa64_ld" #B #N "_acq", \
+ "__aa64_st" #B #N "_rel", \
+ "__aa64_ld" #B #N "_acq_rel" }
+
+const atomic_ool_names aarch64_ool_stadd_names = { { DEF4(add) } };
+const atomic_ool_names aarch64_ool_stset_names = { { DEF4(set) } };
+const atomic_ool_names aarch64_ool_stclr_names = { { DEF4(clr) } };
+const atomic_ool_names aarch64_ool_steor_names = { { DEF4(eor) } };
+
+#undef DEF0
+#undef DEF4
+
/* Expand a compare and swap pattern. */
void
@@ -14204,6 +14288,17 @@ aarch64_expand_compare_and_swap (rtx operands[])
newval, mod_s));
cc_reg = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
}
+ else if (TARGET_ATOMIC_OOL)
+ {
+ /* Oldval must satisfy compare afterward. */
+ if (!aarch64_plus_operand (oldval, mode))
+ oldval = force_reg (mode, oldval);
+ rtx func = aarch64_atomic_ool_func (mode, mod_s, &aarch64_ool_cas_names);
+ rval = emit_library_call_value (func, NULL_RTX, LCT_NORMAL, r_mode,
+ oldval, mode, newval, mode,
+ XEXP (mem, 0), ptr_mode);
+ cc_reg = aarch64_gen_compare_reg_maybe_ze (NE, rval, oldval, mode);
+ }
else
{
/* The oldval predicate varies by mode. Test it and force to reg. */
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf -mno-atomic-ool" } */
#include "atomic-comp-swap-release-acquire.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-acq_rel.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-acquire.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-char.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-consume.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
int v = 0;
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-int.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
long v = 0;
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-relaxed.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-release.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-seq_cst.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "atomic-op-short.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -march=armv8-a+nolse" } */
+/* { dg-options "-O2 -march=armv8-a+nolse -mno-atomic-ool" } */
/* { dg-skip-if "" { *-*-* } { "-mcpu=*" } { "" } } */
int
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-O2 -march=armv8-a+nolse" } */
+/* { dg-options "-O2 -march=armv8-a+nolse -mno-atomic-ool" } */
/* { dg-skip-if "" { *-*-* } { "-mcpu=*" } { "" } } */
int
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -fno-ipa-icf -mno-atomic-ool" } */
#include "sync-comp-swap.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "sync-op-acquire.x"
@@ -1,5 +1,5 @@
/* { dg-do compile } */
-/* { dg-options "-march=armv8-a+nolse -O2" } */
+/* { dg-options "-march=armv8-a+nolse -O2 -mno-atomic-ool" } */
#include "sync-op-full.x"
@@ -218,3 +218,7 @@ Enables verbose cost model dumping in the debug dump files.
mtrack-speculation
Target Var(aarch64_track_speculation)
Generate code to track when the CPU might be speculating incorrectly.
+
+matomic-ool
+Target Report Mask(ATOMIC_OOL) Save
+Generate local calls to out-of-line atomic operations.
@@ -140,16 +140,27 @@
(match_operand:SI 3 "const_int_operand" "")]
""
{
- rtx (*gen) (rtx, rtx, rtx, rtx);
-
/* Use an atomic SWP when available. */
if (TARGET_LSE)
- gen = gen_aarch64_atomic_exchange<mode>_lse;
+ {
+ emit_insn (gen_aarch64_atomic_exchange<mode>_lse
+ (operands[0], operands[1], operands[2], operands[3]));
+ }
+ else if (TARGET_ATOMIC_OOL)
+ {
+ machine_mode mode = <MODE>mode;
+ rtx func = aarch64_atomic_ool_func (mode, operands[3],
+ &aarch64_ool_swp_names);
+ rtx rval = emit_library_call_value (func, operands[0], LCT_NORMAL,
+ mode, operands[2], mode,
+ XEXP (operands[1], 0), ptr_mode);
+ emit_move_insn (operands[0], rval);
+ }
else
- gen = gen_aarch64_atomic_exchange<mode>;
-
- emit_insn (gen (operands[0], operands[1], operands[2], operands[3]));
-
+ {
+ emit_insn (gen_aarch64_atomic_exchange<mode>
+ (operands[0], operands[1], operands[2], operands[3]));
+ }
DONE;
}
)
@@ -234,6 +245,39 @@
}
operands[1] = force_reg (<MODE>mode, operands[1]);
}
+ else if (TARGET_ATOMIC_OOL)
+ {
+ const atomic_ool_names *names;
+ switch (<CODE>)
+ {
+ case MINUS:
+ operands[1] = expand_simple_unop (<MODE>mode, NEG, operands[1],
+ NULL, 1);
+ /* fallthru */
+ case PLUS:
+ names = &aarch64_ool_stadd_names;
+ break;
+ case IOR:
+ names = &aarch64_ool_stset_names;
+ break;
+ case XOR:
+ names = &aarch64_ool_steor_names;
+ break;
+ case AND:
+ operands[1] = expand_simple_unop (<MODE>mode, NOT, operands[1],
+ NULL, 1);
+ names = &aarch64_ool_stclr_names;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ machine_mode mode = <MODE>mode;
+ rtx func = aarch64_atomic_ool_func (mode, operands[2], names);
+ emit_library_call_value (func, NULL_RTX, LCT_NORMAL, mode,
+ operands[1], mode,
+ XEXP (operands[0], 0), ptr_mode);
+ DONE;
+ }
else
gen = gen_aarch64_atomic_<atomic_optab><mode>;
@@ -350,6 +394,40 @@
}
operands[2] = force_reg (<MODE>mode, operands[2]);
}
+ else if (TARGET_ATOMIC_OOL)
+ {
+ const atomic_ool_names *names;
+ switch (<CODE>)
+ {
+ case MINUS:
+ operands[2] = expand_simple_unop (<MODE>mode, NEG, operands[2],
+ NULL, 1);
+ /* fallthru */
+ case PLUS:
+ names = &aarch64_ool_ldadd_names;
+ break;
+ case IOR:
+ names = &aarch64_ool_ldset_names;
+ break;
+ case XOR:
+ names = &aarch64_ool_ldeor_names;
+ break;
+ case AND:
+ operands[2] = expand_simple_unop (<MODE>mode, NOT, operands[2],
+ NULL, 1);
+ names = &aarch64_ool_ldclr_names;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+ machine_mode mode = <MODE>mode;
+ rtx func = aarch64_atomic_ool_func (mode, operands[3], names);
+ rtx rval = emit_library_call_value (func, operands[0], LCT_NORMAL, mode,
+ operands[2], mode,
+ XEXP (operands[1], 0), ptr_mode);
+ emit_move_insn (operands[0], rval);
+ DONE;
+ }
else
gen = gen_aarch64_atomic_fetch_<atomic_optab><mode>;
@@ -439,7 +517,7 @@
{
/* Use an atomic load-operate instruction when possible. In this case
we will re-compute the result from the original mem value. */
- if (TARGET_LSE)
+ if (TARGET_LSE || TARGET_ATOMIC_OOL)
{
rtx tmp = gen_reg_rtx (<MODE>mode);
operands[2] = force_reg (<MODE>mode, operands[2]);
@@ -621,7 +621,7 @@ Objective-C and Objective-C++ Dialects}.
-mpc-relative-literal-loads @gol
-msign-return-address=@var{scope} @gol
-march=@var{name} -mcpu=@var{name} -mtune=@var{name} @gol
--moverride=@var{string} -mverbose-cost-dump -mtrack-speculation}
+-moverride=@var{string} -mverbose-cost-dump -mtrack-speculation -matomic-ool}
@emph{Adapteva Epiphany Options}
@gccoptlist{-mhalf-reg-file -mprefer-short-insn-regs @gol
@@ -15039,6 +15039,18 @@ be used by the compiler when expanding calls to
@code{__builtin_speculation_safe_copy} to permit a more efficient code
sequence to be generated.
+@item -matomic-ool
+@itemx -mno-atomic-ool
+Enable or disable calls to out-of-line helpers to implement atomic operations.
+These helpers will, at runtime, determine if ARMv8.1-Atomics instructions
+should be used; if not, they will use the load/store-exclusive instructions
+that are present in the base ARMv8.0 ISA.
+
+This option is only applicable when compiling for the base ARMv8.0
+instruction set. If using a later revision, e.g. @option{-march=armv8.1-a}
+or @option{-march=armv8-a+lse}, the ARMv8.1-Atomics instructions will be
+used directly.
+
@item -march=@var{name}
@opindex march
Specify the name of the target architecture and, optionally, one or
From: Richard Henderson <richard.henderson@linaro.org> * config/aarch64/aarch64.opt (-matomic-ool): New. * config/aarch64/aarch64.c (aarch64_atomic_ool_func): New. (aarch64_ool_cas_names, aarch64_ool_swp_names): New. (aarch64_ool_ldadd_names, aarch64_ool_ldset_names): New. (aarch64_ool_ldclr_names, aarch64_ool_ldeor_names): New. (aarch64_ool_stadd_names, aarch64_ool_stset_names): New. (aarch64_ool_stclr_names, aarch64_ool_steor_names): New. (aarch64_expand_compare_and_swap): Honor TARGET_ATOMIC_OOL. * config/aarch64/atomics.md (atomic_exchange<ALLI>): Likewise. (atomic_<atomic_op><ALLI>): Likewise. (atomic_fetch_<atomic_op><ALLI>): Likewise. (atomic_<atomic_op>_fetch<ALLI>): Likewise. --- gcc/config/aarch64/aarch64-protos.h | 17 ++++ gcc/config/aarch64/aarch64.c | 95 +++++++++++++++++++ .../atomic-comp-swap-release-acquire.c | 2 +- .../gcc.target/aarch64/atomic-op-acq_rel.c | 2 +- .../gcc.target/aarch64/atomic-op-acquire.c | 2 +- .../gcc.target/aarch64/atomic-op-char.c | 2 +- .../gcc.target/aarch64/atomic-op-consume.c | 2 +- .../gcc.target/aarch64/atomic-op-imm.c | 2 +- .../gcc.target/aarch64/atomic-op-int.c | 2 +- .../gcc.target/aarch64/atomic-op-long.c | 2 +- .../gcc.target/aarch64/atomic-op-relaxed.c | 2 +- .../gcc.target/aarch64/atomic-op-release.c | 2 +- .../gcc.target/aarch64/atomic-op-seq_cst.c | 2 +- .../gcc.target/aarch64/atomic-op-short.c | 2 +- .../aarch64/atomic_cmp_exchange_zero_reg_1.c | 2 +- .../atomic_cmp_exchange_zero_strong_1.c | 2 +- .../gcc.target/aarch64/sync-comp-swap.c | 2 +- .../gcc.target/aarch64/sync-op-acquire.c | 2 +- .../gcc.target/aarch64/sync-op-full.c | 2 +- gcc/config/aarch64/aarch64.opt | 4 + gcc/config/aarch64/atomics.md | 94 ++++++++++++++++-- gcc/doc/invoke.texi | 14 ++- 22 files changed, 232 insertions(+), 26 deletions(-)