diff mbox series

[08/16] malloc: Use global flag instead of function pointer dispatch for mtag

Message ID 8d9711a654bfe7a47ba743ab7e836118f8866f66.1614874816.git.szabolcs.nagy@arm.com
State New
Headers show
Series memory tagging improvements | expand

Commit Message

Szabolcs Nagy March 4, 2021, 4:32 p.m. UTC 
A flag check can be faster than function pointers because of how
branch prediction and speculation works and it can also remove a layer
of indirection when there is a mismatch between the malloc internal
tag_* api and __libc_mtag_* target hooks.

Memory tagging wrapper functions are moved to malloc.c from arena.c and
the logic now checks mmap_enabled.  The definition of tag_new_usable is
moved after chunk related definitions.

This refactoring also allows using mtag_enabled checks instead of
USE_MTAG ifdefs when memory tagging support only changes code logic
when memory tagging is enabled at runtime.
---
 malloc/arena.c  | 33 +---------------------------
 malloc/malloc.c | 58 ++++++++++++++++++++++++++++++++-----------------
 2 files changed, 39 insertions(+), 52 deletions(-)

Comments

Szabolcs Nagy March 5, 2021, 12:53 p.m. UTC | #1 
The 03/04/2021 19:46, DJ Delorie wrote:
> Szabolcs Nagy <szabolcs.nagy@arm.com> writes:
> > The definition of tag_new_usable is moved after chunk related
> > definitions.
> 
> A couple thousand lines after, that is.  Perhaps it could go at the end
> of the chunk related definitions, closer to the other tag functions?

ok. i'm happy to move it around. (i moved it down
close to its first use).

> > This refactoring also allows using mtag_enabled checks instead of
> > USE_MTAG ifdefs when memory tagging support only changes code logic
> > when memory tagging is enabled at runtime.
> 
> Are you relying on the compiler to do the work of the old #ifdefs for
> platforms without USE_MTAG set?  I.e. are we sure that the compiled
> binaries will have the mtag-related code optimized out?

yes as described in my patch 6 response, i attempted to
remove ifdefs.

i think at -O1 we can rely on 'if (0) {...}' to be
optimized out completely (but that code is still type
checked and has to reference defined variables etc.)

with -O0 i'd expect the code to be there in the
compiled libc, just unreachable.

i can add back the ifdefs if this seems too intrusive
(it is now less clear that mtag related code does not
affect the tagging disabled configuration).

> 
> Otherwise LGTM with the above caveats.
> 
> Reviewed-by: DJ Delorie <dj@redhat.com>
diff mbox series

Patch

diff --git a/malloc/arena.c b/malloc/arena.c
index d0778fea92..1e83bb66bd 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -287,34 +287,6 @@  extern struct dl_open_hook *_dl_open_hook;
 libc_hidden_proto (_dl_open_hook);
 #endif
 
-#ifdef USE_MTAG
-
-/* Generate a new (random) tag value for PTR and tag the memory it
-   points to upto the end of the usable size for the chunk containing
-   it.  Return the newly tagged pointer.  */
-static void *
-__mtag_tag_new_usable (void *ptr)
-{
-  if (ptr)
-    {
-      mchunkptr cp = mem2chunk(ptr);
-      ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr),
-				    CHUNK_AVAILABLE_SIZE (cp) - CHUNK_HDR_SZ);
-    }
-  return ptr;
-}
-
-/* Generate a new (random) tag value for PTR, set the tags for the
-   memory to the new tag and initialize the memory contents to VAL.
-   In practice this function will only be called with VAL=0, but we
-   keep this parameter to maintain the same prototype as memset.  */
-static void *
-__mtag_tag_new_memset (void *ptr, int val, size_t size)
-{
-  return __libc_mtag_memset_with_tag (__libc_mtag_new_tag (ptr), val, size);
-}
-#endif
-
 static void
 ptmalloc_init (void)
 {
@@ -332,11 +304,8 @@  ptmalloc_init (void)
       if (__MTAG_SBRK_UNTAGGED)
 	__morecore = __failing_morecore;
 
+      mtag_enabled = true;
       mtag_mmap_flags = __MTAG_MMAP_FLAGS;
-      tag_new_memset = __mtag_tag_new_memset;
-      tag_region = __libc_mtag_tag_region;
-      tag_new_usable = __mtag_tag_new_usable;
-      tag_at = __libc_mtag_address_get_tag;
       mtag_granule_mask = ~(size_t)(__MTAG_GRANULE_SIZE - 1);
     }
 #endif
diff --git a/malloc/malloc.c b/malloc/malloc.c
index e5f520267b..caf34843f7 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -441,35 +441,41 @@  void *(*__morecore)(ptrdiff_t) = __default_morecore;
 */
 
 #ifdef USE_MTAG
+static bool mtag_enabled = false;
+static int mtag_mmap_flags = 0;
+static size_t mtag_granule_mask = ~(size_t)0;
+#else
+# define mtag_enabled false
+# define mtag_mmap_flags 0
+#endif
 
-/* Default implementaions when memory tagging is supported, but disabled.  */
-static void *
-__default_tag_region (void *ptr, size_t size)
+static __always_inline void *
+tag_region (void *ptr, size_t size)
 {
+  if (__glibc_unlikely (mtag_enabled))
+    return __libc_mtag_tag_region (ptr, size);
   return ptr;
 }
 
-static void *
-__default_tag_nop (void *ptr)
+static __always_inline void *
+tag_new_memset (void *ptr, int val, size_t size)
 {
-  return ptr;
+  if (__glibc_unlikely (mtag_enabled))
+    return __libc_mtag_memset_with_tag (__libc_mtag_new_tag (ptr), val, size);
+  return memset (ptr, val, size);
 }
 
-static int mtag_mmap_flags = 0;
-static size_t mtag_granule_mask = ~(size_t)0;
-
-static void *(*tag_new_memset)(void *, int, size_t) = memset;
-static void *(*tag_region)(void *, size_t) = __default_tag_region;
-static void *(*tag_new_usable)(void *) = __default_tag_nop;
-static void *(*tag_at)(void *) = __default_tag_nop;
+/* Defined later.  */
+static void *
+tag_new_usable (void *ptr);
 
-#else
-# define mtag_mmap_flags 0
-# define tag_new_memset(ptr, val, size) memset (ptr, val, size)
-# define tag_region(ptr, size) (ptr)
-# define tag_new_usable(ptr) (ptr)
-# define tag_at(ptr) (ptr)
-#endif
+static __always_inline void *
+tag_at (void *ptr)
+{
+  if (__glibc_unlikely (mtag_enabled))
+    return __libc_mtag_address_get_tag (ptr);
+  return ptr;
+}
 
 #include <string.h>
 
@@ -3185,6 +3191,18 @@  tcache_thread_shutdown (void)
 
 #endif /* !USE_TCACHE  */
 
+static __always_inline void *
+tag_new_usable (void *ptr)
+{
+  if (__glibc_unlikely (mtag_enabled) && ptr)
+    {
+      mchunkptr cp = mem2chunk(ptr);
+      ptr = __libc_mtag_tag_region (__libc_mtag_new_tag (ptr),
+				    CHUNK_AVAILABLE_SIZE (cp) - CHUNK_HDR_SZ);
+    }
+  return ptr;
+}
+
 void *
 __libc_malloc (size_t bytes)
 {