@@ -1832,6 +1832,18 @@ hwasan_memintrin (void)
return (hwasan_sanitize_p () && param_hwasan_instrument_mem_intrinsics);
}
+/* MEMoryTAGging sanitizer (memtag) uses a hardware based capability known as
+ memory tagging to detect memory safety vulnerabilities. Similar to hwasan,
+ it is also a probabilistic method. */
+
+/* Returns whether we are tagging pointers and checking those tags on memory
+ access. */
+bool
+memtag_sanitize_p ()
+{
+ return false;
+}
+
/* Insert code to protect stack vars. The prologue sequence should be emitted
directly, epilogue sequence returned. BASE is the register holding the
stack base, against which OFFSETS array offsets are relative to, OFFSETS
@@ -54,6 +54,8 @@ extern bool hwasan_expand_check_ifn (gimple_stmt_iterator *, bool);
extern bool hwasan_expand_mark_ifn (gimple_stmt_iterator *);
extern bool gate_hwasan (void);
+extern bool memtag_sanitize_p (void);
+
extern gimple_stmt_iterator create_cond_insert_point
(gimple_stmt_iterator *, bool, bool, bool, basic_block *, basic_block *);
@@ -29376,15 +29376,264 @@ aarch64_invalid_binary_op (int op ATTRIBUTE_UNUSED, const_tree type1,
return NULL;
}
+#define AARCH64_MEMTAG_GRANULE_SIZE 16
+#define AARCH64_MEMTAG_TAG_SIZE 4
+
/* Implement TARGET_MEMTAG_CAN_TAG_ADDRESSES. Here we tell the rest of the
compiler that we automatically ignore the top byte of our pointers, which
- allows using -fsanitize=hwaddress. */
+ allows using -fsanitize=hwaddress. In case of -fsanitize=memtag, we
+ additionally ensure that target supports MEMTAG insns. */
bool
aarch64_can_tag_addresses ()
{
+ if (memtag_sanitize_p ())
+ return !TARGET_ILP32 && TARGET_MEMTAG;
return !TARGET_ILP32;
}
+/* Implement TARGET_MEMTAG_TAG_SIZE. */
+unsigned char
+aarch64_memtag_tag_size ()
+{
+ if (memtag_sanitize_p ())
+ return AARCH64_MEMTAG_TAG_SIZE;
+ return default_memtag_tag_size ();
+}
+
+/* Implement TARGET_MEMTAG_GRANULE_SIZE. */
+unsigned char
+aarch64_memtag_granule_size ()
+{
+ if (memtag_sanitize_p ())
+ return AARCH64_MEMTAG_GRANULE_SIZE;
+ return default_memtag_granule_size ();
+}
+
+/* Implement TARGET_MEMTAG_INSERT_RANDOM_TAG. */
+rtx
+aarch64_memtag_insert_random_tag (rtx untagged, rtx target)
+{
+ rtx ret;
+ if (memtag_sanitize_p ())
+ {
+ gcc_assert (param_memtag_instrument_stack || param_memtag_instrument_allocas);
+ if (!target)
+ target = gen_reg_rtx (Pmode);
+
+ rtx insn = gen_irg (target, untagged, untagged);
+ emit_insn (insn);
+
+ ret = XEXP (insn, 0);
+ }
+ else
+ ret = default_memtag_insert_random_tag (untagged, target);
+
+ return ret;
+}
+
+/* Implement TARGET_MEMTAG_ADD_TAG. */
+rtx
+aarch64_memtag_add_tag (rtx base, poly_int64 offset, uint8_t tag_offset)
+{
+ rtx offset_rtx, tag_offset_rtx;
+ rtx target, insn;
+ poly_int64 abs_offset;
+ enum rtx_code code;
+ bool neg_p;
+ rtx ret;
+
+ if (memtag_sanitize_p ())
+ {
+ target = gen_reg_rtx (Pmode);
+ tag_offset_rtx = gen_int_mode (tag_offset, DImode);
+ gcc_assert (aarch64_memtag_tag_offset (tag_offset_rtx, DImode));
+
+ neg_p = known_lt (offset, 0);
+ abs_offset = neg_p ? offset * (-1) : offset;
+ offset_rtx = gen_int_mode (abs_offset, DImode);
+
+ if (!aarch64_granule16_uimm6 (offset_rtx, DImode))
+ {
+ /* Emit addr arithmetic prior to addg/subg. */
+ code = neg_p ? MINUS : PLUS;
+ insn = expand_simple_binop (Pmode, code, base, offset_rtx,
+ target, true, OPTAB_LIB_WIDEN);
+ offset_rtx = const0_rtx;
+ }
+
+ /* Addr offset offset must be within bounds at this time. */
+ gcc_assert (aarch64_granule16_uimm6 (offset_rtx, DImode));
+
+ /* Even if tag_offset_rtx is CONST0_RTX, generate a subg/addg; this
+ provides better opportunity for combining instructions later. */
+ if (neg_p)
+ insn = gen_subg (target, base, offset_rtx, tag_offset_rtx);
+ else
+ insn = gen_addg (target, base, offset_rtx, tag_offset_rtx);
+ emit_insn (insn);
+
+ ret = XEXP (insn, 0);
+ }
+ else
+ ret = default_memtag_add_tag (base, offset, tag_offset);
+
+ return ret;
+}
+
+/* FIXME - Whats a good threshold ? Is there another way to do this? */
+/* Threshold in number of granules beyond which an explicit loop for
+ tagging a memory block is emitted. */
+#define AARCH64_MEMTAG_TAG_MEMORY_LOOP_THRESHOLD 10
+
+static void
+aarch64_memtag_tag_memory_via_loop (rtx base, rtx size, rtx tagged_pointer);
+
+/* The default implementation of TARGET_MEMTAG_TAG_MEMORY. */
+rtx
+aarch64_memtag_tag_memory (rtx base, rtx size, rtx tagged_pointer)
+{
+ rtx stg_rtx;
+ HOST_WIDE_INT factor;
+ HOST_WIDE_INT len, offset;
+ unsigned HOST_WIDE_INT granule_sz;
+ unsigned HOST_WIDE_INT iters;
+
+ granule_sz = (HOST_WIDE_INT) AARCH64_MEMTAG_GRANULE_SIZE;
+
+ /* FIXME check predicate on offset (from base) + size. */
+ if (CONST_INT_P (size) && aarch64_granule16_simm9 (size, DImode))
+ {
+ len = INTVAL (size);
+ /* The amount of memory to tag must be aligned to granule size by now. */
+ gcc_assert (abs_hwi (len) % granule_sz == 0);
+
+ factor = (known_le (len, 0)) ? -1 : 1;
+ iters = abs_hwi (len) / granule_sz;
+
+ offset = 0;
+
+ if (iters > AARCH64_MEMTAG_TAG_MEMORY_LOOP_THRESHOLD)
+ goto emit_loop_tag_memory;
+
+ /* gen_stg / gen_st2g expects a simple PLUS (reg, offset) as addr operand. */
+ if (!REG_P (base))
+ {
+ rtx addr = simplify_gen_binary (PLUS, Pmode, base,
+ gen_int_mode (offset, Pmode));
+ if (!CONST_INT_P (XEXP (addr, 1)))
+ {
+ emit_insn (addr);
+ base = XEXP (addr, 0);
+ }
+ else
+ {
+ base = XEXP (addr, 0);
+ offset += INTVAL (XEXP (addr, 1));
+ }
+ }
+
+ while (iters)
+ {
+ if (iters / 2)
+ {
+ stg_rtx = gen_st2g (tagged_pointer, base,
+ gen_int_mode (offset, Pmode),
+ gen_int_mode (offset - 16, Pmode));
+ iters -= 2;
+ }
+ else
+ {
+ stg_rtx = gen_stg (tagged_pointer, base, gen_int_mode (offset, Pmode));
+ iters -= 1;
+ }
+
+ emit_insn (stg_rtx);
+ offset = granule_sz * iters * factor;
+ }
+
+ return stg_rtx;
+ }
+
+emit_loop_tag_memory:
+ /* FIXME - stg_rtx to be returned. Update signature to return void / bool ? */
+ stg_rtx = NULL;
+ aarch64_memtag_tag_memory_via_loop (base, size, tagged_pointer);
+
+ return stg_rtx;
+}
+
+/* Tag the memory via an explicit loop. This is used when target hook
+ TARGET_MEMTAG_TAG_MEMORY is invoked for non-constant size / constant but not
+ encodable size / when (constant) size is over a threshold. */
+static void
+aarch64_memtag_tag_memory_via_loop (rtx base, rtx size, rtx tagged_pointer)
+{
+ rtx_code_label *top_label;
+ rtx iter, x_addr, tmp;
+ machine_mode iter_mode;
+ unsigned HOST_WIDE_INT len;
+ unsigned HOST_WIDE_INT granule_sz;
+ unsigned HOST_WIDE_INT iters;
+ granule_sz = (HOST_WIDE_INT) AARCH64_MEMTAG_GRANULE_SIZE;
+
+ unsigned int factor = 1;
+ machine_mode x_addr_mode = GET_MODE (base);
+
+ iter_mode = GET_MODE (size);
+ if (iter_mode == VOIDmode)
+ iter_mode = word_mode;
+
+ if (CONST_INT_P (size))
+ {
+ len = INTVAL (size);
+ /* The amount of memory to tag must be aligned to granule size by now. */
+ gcc_assert (abs_hwi (len) % granule_sz == 0);
+ iters = abs_hwi (len) / granule_sz;
+ /* st2g may be used to tag two granules in the same instruction. */
+ if (iters % 2 == 0)
+ factor = 2;
+ }
+
+ top_label = gen_label_rtx ();
+ x_addr = base;
+
+ /* Forward direction loop with increment equal to a non zero
+ multiple of granule size. */
+ rtx iter_init = const0_rtx;
+ rtx_code iter_cond = LTU;
+ rtx iter_limit = size;
+ rtx iter_incr = GEN_INT (granule_sz * factor);
+ /* Emit ITER. */
+ iter = gen_reg_rtx (iter_mode);
+ emit_move_insn (iter, iter_init);
+
+ /* Emit top label. */
+ emit_label (top_label);
+
+ /* Offset the base addresses by ITER. */
+ tmp = convert_modes (x_addr_mode, iter_mode, iter, true);
+
+ rtx addr_reg = gen_reg_rtx (Pmode);
+ tmp = expand_simple_binop (x_addr_mode, PLUS, addr_reg, x_addr, tmp,
+ true, OPTAB_LIB_WIDEN);
+ if (tmp != addr_reg)
+ emit_set_insn (addr_reg, tmp);
+
+ /* Tag Memory. */
+ aarch64_memtag_tag_memory (addr_reg, iter_incr, tagged_pointer);
+
+ /* Increment ITER. */
+ tmp = expand_simple_binop (iter_mode, PLUS, iter, iter_incr, iter,
+ true, OPTAB_LIB_WIDEN);
+ if (tmp != iter)
+ emit_move_insn (iter, tmp);
+
+ emit_cmp_and_jump_insns (iter, iter_limit, iter_cond, NULL_RTX, iter_mode,
+ true, top_label,
+ profile_probability::guessed_always ()
+ .apply_scale (9, 10));
+}
+
/* Implement TARGET_ASM_FILE_END for AArch64. This adds the AArch64 GNU NOTE
section at the end if needed. */
#define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
@@ -31455,6 +31704,21 @@ aarch64_libgcc_floating_mode_supported_p
#undef TARGET_MEMTAG_CAN_TAG_ADDRESSES
#define TARGET_MEMTAG_CAN_TAG_ADDRESSES aarch64_can_tag_addresses
+#undef TARGET_MEMTAG_TAG_SIZE
+#define TARGET_MEMTAG_TAG_SIZE aarch64_memtag_tag_size
+
+#undef TARGET_MEMTAG_GRANULE_SIZE
+#define TARGET_MEMTAG_GRANULE_SIZE aarch64_memtag_granule_size
+
+#undef TARGET_MEMTAG_INSERT_RANDOM_TAG
+#define TARGET_MEMTAG_INSERT_RANDOM_TAG aarch64_memtag_insert_random_tag
+
+#undef TARGET_MEMTAG_ADD_TAG
+#define TARGET_MEMTAG_ADD_TAG aarch64_memtag_add_tag
+
+#undef TARGET_MEMTAG_TAG_MEMORY
+#define TARGET_MEMTAG_TAG_MEMORY aarch64_memtag_tag_memory
+
#if CHECKING_P
#undef TARGET_RUN_TARGET_SELFTESTS
#define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests