| Message ID | 20231222165830.2100438-2-hjl.tools@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | x86/cet: Update CET kernel interface | expand |
On Fri, Dec 22, 2023 at 8:58 AM H.J. Lu <hjl.tools@gmail.com> wrote: > > Sync with Linux kernel 6.6 shadow stack interface. Since only x86-64 is > supported, i386 shadow stack codes are unchanged and CET shouldn't be > enabled for i386. > > 1. When the shadow stack base in TCB is unset, the default shadow stack > is in use. Use the current shadow stack pointer as the marker for the > default shadow stack. It is used to identify if the current shadow stack > is the same as the target shadow stack when switching ucontexts. If yes, > INCSSP will be used to unwind shadow stack. Otherwise, shadow stack > restore token will be used. > 2. Allocate shadow stack with the map_shadow_stack syscall. Since there > is no function to explicitly release ucontext, there is no place to > release shadow stack allocated by map_shadow_stack in ucontext functions. > Such shadow stacks will be leaked. > 3. Rename arch_prctl CET commands to ARCH_SHSTK_XXX. > 4. Rewrite the CET control functions with the current kernel shadow stack > interface. > > Since CET is no longer enabled by kernel, a separate patch will enable > shadow stack during startup. > --- > sysdeps/unix/sysv/linux/x86/bits/mman.h | 5 ++ > sysdeps/unix/sysv/linux/x86/cpu-features.c | 13 +++-- > sysdeps/unix/sysv/linux/x86/dl-cet.h | 16 ++++-- > .../unix/sysv/linux/x86/include/asm/prctl.h | 37 ++++++------- > .../sysv/linux/x86/tst-cet-setcontext-1.c | 17 +++--- > sysdeps/unix/sysv/linux/x86_64/Makefile | 2 +- > .../unix/sysv/linux/x86_64/__start_context.S | 38 +++---------- > .../sysv/linux/x86_64/allocate-shadow-stack.c | 55 +++++++++++++++++++ > .../sysv/linux/x86_64/allocate-shadow-stack.h | 24 ++++++++ > sysdeps/unix/sysv/linux/x86_64/getcontext.S | 30 ++-------- > sysdeps/unix/sysv/linux/x86_64/makecontext.c | 28 +++++----- > sysdeps/unix/sysv/linux/x86_64/swapcontext.S | 22 ++------ > sysdeps/x86/cpu-features.c | 15 +++-- > sysdeps/x86/dl-cet.c | 2 +- > sysdeps/x86_64/nptl/tls.h | 2 +- > 15 files changed, 173 insertions(+), 133 deletions(-) > create mode 100644 sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > create mode 100644 sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > > diff --git a/sysdeps/unix/sysv/linux/x86/bits/mman.h b/sysdeps/unix/sysv/linux/x86/bits/mman.h > index 3d356e86a0..232b55a13d 100644 > --- a/sysdeps/unix/sysv/linux/x86/bits/mman.h > +++ b/sysdeps/unix/sysv/linux/x86/bits/mman.h > @@ -27,6 +27,11 @@ > #define MAP_32BIT 0x40 /* Only give out 32-bit addresses. */ > #define MAP_ABOVE4G 0x80 /* Only map above 4GB. */ > > +#ifdef __USE_MISC > +/* Set up a restore token in the newly allocated shadow stack */ > +# define SHADOW_STACK_SET_TOKEN 0x1 > +#endif > + > #include <bits/mman-map-flags-generic.h> > > /* Include generic Linux declarations. */ > diff --git a/sysdeps/unix/sysv/linux/x86/cpu-features.c b/sysdeps/unix/sysv/linux/x86/cpu-features.c > index 41e7600668..0e6e2bf855 100644 > --- a/sysdeps/unix/sysv/linux/x86/cpu-features.c > +++ b/sysdeps/unix/sysv/linux/x86/cpu-features.c > @@ -23,10 +23,15 @@ > static inline int __attribute__ ((always_inline)) > get_cet_status (void) > { > - unsigned long long cet_status[3]; > - if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_STATUS, cet_status) == 0) > - return cet_status[0]; > - return 0; > + unsigned long long kernel_feature; > + unsigned int status = 0; > + if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS, > + &kernel_feature) == 0) > + { > + if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0) > + status = GNU_PROPERTY_X86_FEATURE_1_SHSTK; > + } > + return status; > } > > # ifndef SHARED > diff --git a/sysdeps/unix/sysv/linux/x86/dl-cet.h b/sysdeps/unix/sysv/linux/x86/dl-cet.h > index c885bf1323..da220ac627 100644 > --- a/sysdeps/unix/sysv/linux/x86/dl-cet.h > +++ b/sysdeps/unix/sysv/linux/x86/dl-cet.h > @@ -21,12 +21,20 @@ > static inline int __attribute__ ((always_inline)) > dl_cet_disable_cet (unsigned int cet_feature) > { > - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_DISABLE, > - cet_feature); > + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) > + return -1; > + long long int kernel_feature = ARCH_SHSTK_SHSTK; > + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_DISABLE, > + kernel_feature); > } > > static inline int __attribute__ ((always_inline)) > -dl_cet_lock_cet (void) > +dl_cet_lock_cet (unsigned int cet_feature) > { > - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_LOCK, 0); > + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) > + return -1; > + /* Lock all SHSTK features. */ > + long long int kernel_feature = -1; > + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK, > + kernel_feature); > } > diff --git a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > index 45ad0b052f..2f511321ad 100644 > --- a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > +++ b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > @@ -4,24 +4,19 @@ > > #include_next <asm/prctl.h> > > -#ifndef ARCH_CET_STATUS > -/* CET features: > - IBT: GNU_PROPERTY_X86_FEATURE_1_IBT > - SHSTK: GNU_PROPERTY_X86_FEATURE_1_SHSTK > - */ > -/* Return CET features in unsigned long long *addr: > - features: addr[0]. > - shadow stack base address: addr[1]. > - shadow stack size: addr[2]. > - */ > -# define ARCH_CET_STATUS 0x3001 > -/* Disable CET features in unsigned int features. */ > -# define ARCH_CET_DISABLE 0x3002 > -/* Lock all CET features. */ > -# define ARCH_CET_LOCK 0x3003 > -/* Allocate a new shadow stack with unsigned long long *addr: > - IN: requested shadow stack size: *addr. > - OUT: allocated shadow stack address: *addr. > - */ > -# define ARCH_CET_ALLOC_SHSTK 0x3004 > -#endif /* ARCH_CET_STATUS */ > +#ifndef ARCH_SHSTK_ENABLE > +/* Enable SHSTK features in unsigned long int features. */ > +# define ARCH_SHSTK_ENABLE 0x5001 > +/* Disable SHSTK features in unsigned long int features. */ > +# define ARCH_SHSTK_DISABLE 0x5002 > +/* Lock SHSTK features in unsigned long int features. */ > +# define ARCH_SHSTK_LOCK 0x5003 > +/* Unlock SHSTK features in unsigned long int features. */ > +# define ARCH_SHSTK_UNLOCK 0x5004 > +/* Return SHSTK features in unsigned long int features. */ > +# define ARCH_SHSTK_STATUS 0x5005 > + > +/* ARCH_SHSTK_ features bits */ > +# define ARCH_SHSTK_SHSTK 0x1 > +# define ARCH_SHSTK_WRSS 0x2 > +#endif > diff --git a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > index 837a9fd0eb..2ea66c803b 100644 > --- a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > +++ b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > @@ -87,15 +87,14 @@ do_test (void) > ctx[4].uc_link = &ctx[0]; > makecontext (&ctx[4], (void (*) (void)) f1, 0); > > - /* NB: When shadow stack is enabled, makecontext calls arch_prctl > - with ARCH_CET_ALLOC_SHSTK to allocate a new shadow stack which > - can be unmapped. The base address and size of the new shadow > - stack are returned in __ssp[1] and __ssp[2]. makecontext is > - called for CTX1, CTX3 and CTX4. But only CTX1 is used. New > - shadow stacks are allocated in the order of CTX3, CTX1, CTX4. > - It is very likely that CTX1's shadow stack is placed between > - CTX3 and CTX4. We munmap CTX3's and CTX4's shadow stacks to > - create gaps above and below CTX1's shadow stack. We check > + /* NB: When shadow stack is enabled, makecontext calls map_shadow_stack > + to allocate a new shadow stack which can be unmapped. The base > + address and size of the new shadow stack are returned in __ssp[1] > + and __ssp[2]. makecontext is called for CTX1, CTX3 and CTX4. But > + only CTX1 is used. New shadow stacks are allocated in the order > + of CTX3, CTX1, CTX4. It is very likely that CTX1's shadow stack is > + placed between CTX3 and CTX4. We munmap CTX3's and CTX4's shadow > + stacks to create gaps above and below CTX1's shadow stack. We check > that setcontext CTX1 works correctly in this case. */ > if (_get_ssp () != 0) > { > diff --git a/sysdeps/unix/sysv/linux/x86_64/Makefile b/sysdeps/unix/sysv/linux/x86_64/Makefile > index 5e19202ebf..06b873949e 100644 > --- a/sysdeps/unix/sysv/linux/x86_64/Makefile > +++ b/sysdeps/unix/sysv/linux/x86_64/Makefile > @@ -3,7 +3,7 @@ sysdep_routines += ioperm iopl > endif > > ifeq ($(subdir),stdlib) > -sysdep_routines += __start_context > +sysdep_routines += __start_context allocate-shadow-stack > endif > > ifeq ($(subdir),csu) > diff --git a/sysdeps/unix/sysv/linux/x86_64/__start_context.S b/sysdeps/unix/sysv/linux/x86_64/__start_context.S > index f6436dd6bb..ae04203c90 100644 > --- a/sysdeps/unix/sysv/linux/x86_64/__start_context.S > +++ b/sysdeps/unix/sysv/linux/x86_64/__start_context.S > @@ -24,20 +24,14 @@ > /* Use CALL to push __start_context onto the new stack as well as the new > shadow stack. RDI points to ucontext: > Incoming: > - __ssp[0]: The original caller's shadow stack pointer. > - __ssp[1]: The size of the new shadow stack. > - __ssp[2]: The size of the new shadow stack. > - Outgoing: > __ssp[0]: The new shadow stack pointer. > __ssp[1]: The base address of the new shadow stack. > __ssp[2]: The size of the new shadow stack. > */ > > ENTRY(__push___start_context) > - /* Save the pointer to ucontext. */ > - movq %rdi, %r9 > /* Get the original shadow stack pointer. */ > - rdsspq %r8 > + rdsspq %rcx > /* Save the original stack pointer. */ > movq %rsp, %rdx > /* Load the top of the new stack into RSI. */ > @@ -45,24 +39,12 @@ ENTRY(__push___start_context) > /* Add 8 bytes to RSI since CALL will push the 8-byte return > address onto stack. */ > leaq 8(%rsi), %rsp > - /* Allocate the new shadow stack. The size of the new shadow > - stack is passed in __ssp[1]. */ > - lea (oSSP + 8)(%rdi), %RSI_LP > - movl $ARCH_CET_ALLOC_SHSTK, %edi > - movl $__NR_arch_prctl, %eax > - /* The new shadow stack base is returned in __ssp[1]. */ > - syscall > - testq %rax, %rax > - jne L(hlt) /* This should never happen. */ > - > - /* Get the size of the new shadow stack. */ > - movq 8(%rsi), %rdi > - > - /* Get the base address of the new shadow stack. */ > - movq (%rsi), %rsi > - > + /* The size of the new shadow stack is stored in __ssp[2]. */ > + mov (oSSP + 16)(%rdi), %RSI_LP > + /* The new shadow stack base is stored in __ssp[1]. */ > + mov (oSSP + 8)(%rdi), %RAX_LP > /* Use the restore stoken to restore the new shadow stack. */ > - rstorssp -8(%rsi, %rdi) > + rstorssp -8(%rax, %rsi) > > /* Save the restore token on the original shadow stack. */ > saveprevssp > @@ -73,18 +55,12 @@ ENTRY(__push___start_context) > jmp __start_context > 1: > > - /* Get the new shadow stack pointer. */ > - rdsspq %rdi > - > /* Use the restore stoken to restore the original shadow stack. */ > - rstorssp -8(%r8) > + rstorssp -8(%rcx) > > /* Save the restore token on the new shadow stack. */ > saveprevssp > > - /* Store the new shadow stack pointer in __ssp[0]. */ > - movq %rdi, oSSP(%r9) > - > /* Restore the original stack. */ > mov %rdx, %rsp > ret > diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > new file mode 100644 > index 0000000000..f2e1d03b96 > --- /dev/null > +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > @@ -0,0 +1,55 @@ > +/* Helper function to allocate shadow stack. > + Copyright (C) 2023 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#include <sysdep.h> > +#include <stdint.h> > +#include <errno.h> > +#include <sys/mman.h> > +#include <libc-pointer-arith.h> > +#include <allocate-shadow-stack.h> > + > +/* NB: This can be treated as a syscall by caller. */ > + > +long int > +__allocate_shadow_stack (size_t stack_size, > + shadow_stack_size_t *child_stack) > +{ > +#ifdef __NR_map_shadow_stack > + size_t shadow_stack_size > + = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT; > + /* Align shadow stack to 8 bytes. */ > + shadow_stack_size = ALIGN_UP (shadow_stack_size, 8); > + /* Since sigaltstack shares shadow stack with the current context in > + the thread, add extra 20 stack frames in shadow stack for signal > + handlers. */ > + shadow_stack_size += 20 * 8; > + void *shadow_stack = (void *)INLINE_SYSCALL_CALL > + (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN); > + /* Report the map_shadow_stack error. */ > + if (shadow_stack == MAP_FAILED) > + return -errno; > + > + /* Save the shadow stack base and size on child stack. */ > + child_stack[0] = (uintptr_t) shadow_stack; > + child_stack[1] = shadow_stack_size; > + > + return 0; > +#else > + return -ENOSYS; > +#endif > +} > diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > new file mode 100644 > index 0000000000..d05aaf16e5 > --- /dev/null > +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > @@ -0,0 +1,24 @@ > +/* Helper function to allocate shadow stack. > + Copyright (C) 2023 Free Software Foundation, Inc. > + This file is part of the GNU C Library. > + > + The GNU C Library is free software; you can redistribute it and/or > + modify it under the terms of the GNU Lesser General Public > + License as published by the Free Software Foundation; either > + version 2.1 of the License, or (at your option) any later version. > + > + The GNU C Library is distributed in the hope that it will be useful, > + but WITHOUT ANY WARRANTY; without even the implied warranty of > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > + Lesser General Public License for more details. > + > + You should have received a copy of the GNU Lesser General Public > + License along with the GNU C Library; if not, see > + <https://www.gnu.org/licenses/>. */ > + > +#include <ucontext.h> > + > +typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t; > + > +extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *) > + attribute_hidden; > diff --git a/sysdeps/unix/sysv/linux/x86_64/getcontext.S b/sysdeps/unix/sysv/linux/x86_64/getcontext.S > index a00e2f6290..71f3802dca 100644 > --- a/sysdeps/unix/sysv/linux/x86_64/getcontext.S > +++ b/sysdeps/unix/sysv/linux/x86_64/getcontext.S > @@ -58,35 +58,15 @@ ENTRY(__getcontext) > testl $X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET > jz L(no_shstk) > > - /* Save RDI in RDX which won't be clobbered by syscall. */ > - movq %rdi, %rdx > - > xorl %eax, %eax > cmpq %fs:SSP_BASE_OFFSET, %rax > jnz L(shadow_stack_bound_recorded) > > - /* Get the base address and size of the default shadow stack > - which must be the current shadow stack since nothing has > - been recorded yet. */ > - sub $24, %RSP_LP > - mov %RSP_LP, %RSI_LP > - movl $ARCH_CET_STATUS, %edi > - movl $__NR_arch_prctl, %eax > - syscall > - testq %rax, %rax > - jz L(continue_no_err) > - > - /* This should never happen. */ > - hlt > - > -L(continue_no_err): > - /* Record the base of the current shadow stack. */ > - movq 8(%rsp), %rax > + /* When the shadow stack base is unset, the default shadow > + stack is in use. Use the current shadow stack pointer > + as the marker for the default shadow stack. */ > + rdsspq %rax > movq %rax, %fs:SSP_BASE_OFFSET > - add $24, %RSP_LP > - > - /* Restore RDI. */ > - movq %rdx, %rdi > > L(shadow_stack_bound_recorded): > /* Get the current shadow stack pointer. */ > @@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded): > /* NB: Save the caller's shadow stack so that we can jump back > to the caller directly. */ > addq $8, %rax > - movq %rax, oSSP(%rdx) > + movq %rax, oSSP(%rdi) > > /* Save the current shadow stack base in ucontext. */ > movq %fs:SSP_BASE_OFFSET, %rax > diff --git a/sysdeps/unix/sysv/linux/x86_64/makecontext.c b/sysdeps/unix/sysv/linux/x86_64/makecontext.c > index de9e03eb81..e4f025bd50 100644 > --- a/sysdeps/unix/sysv/linux/x86_64/makecontext.c > +++ b/sysdeps/unix/sysv/linux/x86_64/makecontext.c > @@ -24,6 +24,7 @@ > # include <pthread.h> > # include <libc-pointer-arith.h> > # include <sys/prctl.h> > +# include <allocate-shadow-stack.h> > #endif > > #include "ucontext_i.h" > @@ -88,23 +89,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...) > if ((feature_1 & X86_FEATURE_1_SHSTK) != 0) > { > /* Shadow stack is enabled. We need to allocate a new shadow > - stack. */ > - unsigned long ssp_size = (((uintptr_t) sp > - - (uintptr_t) ucp->uc_stack.ss_sp) > - >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT); > - /* Align shadow stack to 8 bytes. */ > - ssp_size = ALIGN_UP (ssp_size, 8); > - > - ucp->__ssp[1] = ssp_size; > - ucp->__ssp[2] = ssp_size; > - > - /* Call __push___start_context to allocate a new shadow stack, > - push __start_context onto the new stack as well as the new > - shadow stack. NB: After __push___start_context returns, > + stack. NB: > ucp->__ssp[0]: The new shadow stack pointer. > ucp->__ssp[1]: The base address of the new shadow stack. > ucp->__ssp[2]: The size of the new shadow stack. > */ > + long int ret > + = __allocate_shadow_stack (((uintptr_t) sp > + - (uintptr_t) ucp->uc_stack.ss_sp), > + &ucp->__ssp[1]); > + if (ret != 0) > + { > + /* FIXME: What should we do? */ > + abort (); > + } > + > + ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8; > + /* Call __push___start_context to push __start_context onto the new > + stack as well as the new shadow stack. */ > __push___start_context (ucp); > } > else > diff --git a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > index 5925752164..2f2fe9875b 100644 > --- a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > +++ b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > @@ -109,25 +109,11 @@ ENTRY(__swapcontext) > cmpq %fs:SSP_BASE_OFFSET, %rax > jnz L(shadow_stack_bound_recorded) > > - /* Get the base address and size of the default shadow stack > - which must be the current shadow stack since nothing has > - been recorded yet. */ > - sub $24, %RSP_LP > - mov %RSP_LP, %RSI_LP > - movl $ARCH_CET_STATUS, %edi > - movl $__NR_arch_prctl, %eax > - syscall > - testq %rax, %rax > - jz L(continue_no_err) > - > - /* This should never happen. */ > - hlt > - > -L(continue_no_err): > - /* Record the base of the current shadow stack. */ > - movq 8(%rsp), %rax > + /* When the shadow stack base is unset, the default shadow > + stack is in use. Use the current shadow stack pointer > + as the marker for the default shadow stack. */ > + rdsspq %rax > movq %rax, %fs:SSP_BASE_OFFSET > - add $24, %RSP_LP > > L(shadow_stack_bound_recorded): > /* If we unwind the stack, we can't undo stack unwinding. Just > diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c > index 0bf923d48b..f180f0d9a4 100644 > --- a/sysdeps/x86/cpu-features.c > +++ b/sysdeps/x86/cpu-features.c > @@ -1121,8 +1121,9 @@ no_cpuid: > > # ifndef SHARED > /* Check if IBT and SHSTK are enabled by kernel. */ > - if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT) > - || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK)) > + if ((cet_status > + & (GNU_PROPERTY_X86_FEATURE_1_IBT > + | GNU_PROPERTY_X86_FEATURE_1_SHSTK))) I think the code here and elsewhere would be simplifiable with a define/enum of `GNU_PROPERTY_X86_FEATURE_1_SHSTK_OR_IBT = GNU_PROPERTY_X86_FEATURE_1_SHSTK | GNU_PROPERTY_X86_FEATURE_1_IBT` > { > /* Disable IBT and/or SHSTK if they are enabled by kernel, but > disabled by environment variable: > @@ -1131,9 +1132,11 @@ no_cpuid: > */ > unsigned int cet_feature = 0; > if (!CPU_FEATURE_USABLE (IBT)) > - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT; > + cet_feature |= (cet_status > + & GNU_PROPERTY_X86_FEATURE_1_IBT); > if (!CPU_FEATURE_USABLE (SHSTK)) > - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; > + cet_feature |= (cet_status > + & GNU_PROPERTY_X86_FEATURE_1_SHSTK); > > if (cet_feature) > { > @@ -1148,7 +1151,9 @@ no_cpuid: > lock CET if IBT or SHSTK is enabled permissively. */ > if (GL(dl_x86_feature_control).ibt != cet_permissive > && GL(dl_x86_feature_control).shstk != cet_permissive) > - dl_cet_lock_cet (); > + dl_cet_lock_cet (GL(dl_x86_feature_1) > + & (GNU_PROPERTY_X86_FEATURE_1_IBT > + | GNU_PROPERTY_X86_FEATURE_1_SHSTK)); > } > # endif > } > diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c > index e486e549be..66a78244d4 100644 > --- a/sysdeps/x86/dl-cet.c > +++ b/sysdeps/x86/dl-cet.c > @@ -202,7 +202,7 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info) > feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; > > if (feature_1_lock != 0 > - && dl_cet_lock_cet () != 0) > + && dl_cet_lock_cet (feature_1_lock) != 0) > _dl_fatal_printf ("%s: can't lock CET\n", info->program); > } > > diff --git a/sysdeps/x86_64/nptl/tls.h b/sysdeps/x86_64/nptl/tls.h > index 1403f939f7..4bcc2552a1 100644 > --- a/sysdeps/x86_64/nptl/tls.h > +++ b/sysdeps/x86_64/nptl/tls.h > @@ -60,7 +60,7 @@ typedef struct > void *__private_tm[4]; > /* GCC split stack support. */ > void *__private_ss; > - /* The lowest address of shadow stack, */ > + /* The marker for the current shadow stack. */ > unsigned long long int ssp_base; > /* Must be kept even if it is no longer used by glibc since programs, > like AddressSanitizer, depend on the size of tcbhead_t. */ > -- > 2.43.0 >
On Tue, Dec 26, 2023 at 9:38 AM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > On Fri, Dec 22, 2023 at 8:58 AM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > Sync with Linux kernel 6.6 shadow stack interface. Since only x86-64 is > > supported, i386 shadow stack codes are unchanged and CET shouldn't be > > enabled for i386. > > > > 1. When the shadow stack base in TCB is unset, the default shadow stack > > is in use. Use the current shadow stack pointer as the marker for the > > default shadow stack. It is used to identify if the current shadow stack > > is the same as the target shadow stack when switching ucontexts. If yes, > > INCSSP will be used to unwind shadow stack. Otherwise, shadow stack > > restore token will be used. > > 2. Allocate shadow stack with the map_shadow_stack syscall. Since there > > is no function to explicitly release ucontext, there is no place to > > release shadow stack allocated by map_shadow_stack in ucontext functions. > > Such shadow stacks will be leaked. > > 3. Rename arch_prctl CET commands to ARCH_SHSTK_XXX. > > 4. Rewrite the CET control functions with the current kernel shadow stack > > interface. > > > > Since CET is no longer enabled by kernel, a separate patch will enable > > shadow stack during startup. > > --- > > sysdeps/unix/sysv/linux/x86/bits/mman.h | 5 ++ > > sysdeps/unix/sysv/linux/x86/cpu-features.c | 13 +++-- > > sysdeps/unix/sysv/linux/x86/dl-cet.h | 16 ++++-- > > .../unix/sysv/linux/x86/include/asm/prctl.h | 37 ++++++------- > > .../sysv/linux/x86/tst-cet-setcontext-1.c | 17 +++--- > > sysdeps/unix/sysv/linux/x86_64/Makefile | 2 +- > > .../unix/sysv/linux/x86_64/__start_context.S | 38 +++---------- > > .../sysv/linux/x86_64/allocate-shadow-stack.c | 55 +++++++++++++++++++ > > .../sysv/linux/x86_64/allocate-shadow-stack.h | 24 ++++++++ > > sysdeps/unix/sysv/linux/x86_64/getcontext.S | 30 ++-------- > > sysdeps/unix/sysv/linux/x86_64/makecontext.c | 28 +++++----- > > sysdeps/unix/sysv/linux/x86_64/swapcontext.S | 22 ++------ > > sysdeps/x86/cpu-features.c | 15 +++-- > > sysdeps/x86/dl-cet.c | 2 +- > > sysdeps/x86_64/nptl/tls.h | 2 +- > > 15 files changed, 173 insertions(+), 133 deletions(-) > > create mode 100644 sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > > create mode 100644 sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > > > > diff --git a/sysdeps/unix/sysv/linux/x86/bits/mman.h b/sysdeps/unix/sysv/linux/x86/bits/mman.h > > index 3d356e86a0..232b55a13d 100644 > > --- a/sysdeps/unix/sysv/linux/x86/bits/mman.h > > +++ b/sysdeps/unix/sysv/linux/x86/bits/mman.h > > @@ -27,6 +27,11 @@ > > #define MAP_32BIT 0x40 /* Only give out 32-bit addresses. */ > > #define MAP_ABOVE4G 0x80 /* Only map above 4GB. */ > > > > +#ifdef __USE_MISC > > +/* Set up a restore token in the newly allocated shadow stack */ > > +# define SHADOW_STACK_SET_TOKEN 0x1 > > +#endif > > + > > #include <bits/mman-map-flags-generic.h> > > > > /* Include generic Linux declarations. */ > > diff --git a/sysdeps/unix/sysv/linux/x86/cpu-features.c b/sysdeps/unix/sysv/linux/x86/cpu-features.c > > index 41e7600668..0e6e2bf855 100644 > > --- a/sysdeps/unix/sysv/linux/x86/cpu-features.c > > +++ b/sysdeps/unix/sysv/linux/x86/cpu-features.c > > @@ -23,10 +23,15 @@ > > static inline int __attribute__ ((always_inline)) > > get_cet_status (void) > > { > > - unsigned long long cet_status[3]; > > - if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_STATUS, cet_status) == 0) > > - return cet_status[0]; > > - return 0; > > + unsigned long long kernel_feature; > > + unsigned int status = 0; > > + if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS, > > + &kernel_feature) == 0) > > + { > > + if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0) > > + status = GNU_PROPERTY_X86_FEATURE_1_SHSTK; > > + } > > + return status; > > } > > > > # ifndef SHARED > > diff --git a/sysdeps/unix/sysv/linux/x86/dl-cet.h b/sysdeps/unix/sysv/linux/x86/dl-cet.h > > index c885bf1323..da220ac627 100644 > > --- a/sysdeps/unix/sysv/linux/x86/dl-cet.h > > +++ b/sysdeps/unix/sysv/linux/x86/dl-cet.h > > @@ -21,12 +21,20 @@ > > static inline int __attribute__ ((always_inline)) > > dl_cet_disable_cet (unsigned int cet_feature) > > { > > - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_DISABLE, > > - cet_feature); > > + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) > > + return -1; > > + long long int kernel_feature = ARCH_SHSTK_SHSTK; > > + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_DISABLE, > > + kernel_feature); > > } > > > > static inline int __attribute__ ((always_inline)) > > -dl_cet_lock_cet (void) > > +dl_cet_lock_cet (unsigned int cet_feature) > > { > > - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_LOCK, 0); > > + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) > > + return -1; > > + /* Lock all SHSTK features. */ > > + long long int kernel_feature = -1; > > + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK, > > + kernel_feature); > > } > > diff --git a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > > index 45ad0b052f..2f511321ad 100644 > > --- a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > > +++ b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > > @@ -4,24 +4,19 @@ > > > > #include_next <asm/prctl.h> > > > > -#ifndef ARCH_CET_STATUS > > -/* CET features: > > - IBT: GNU_PROPERTY_X86_FEATURE_1_IBT > > - SHSTK: GNU_PROPERTY_X86_FEATURE_1_SHSTK > > - */ > > -/* Return CET features in unsigned long long *addr: > > - features: addr[0]. > > - shadow stack base address: addr[1]. > > - shadow stack size: addr[2]. > > - */ > > -# define ARCH_CET_STATUS 0x3001 > > -/* Disable CET features in unsigned int features. */ > > -# define ARCH_CET_DISABLE 0x3002 > > -/* Lock all CET features. */ > > -# define ARCH_CET_LOCK 0x3003 > > -/* Allocate a new shadow stack with unsigned long long *addr: > > - IN: requested shadow stack size: *addr. > > - OUT: allocated shadow stack address: *addr. > > - */ > > -# define ARCH_CET_ALLOC_SHSTK 0x3004 > > -#endif /* ARCH_CET_STATUS */ > > +#ifndef ARCH_SHSTK_ENABLE > > +/* Enable SHSTK features in unsigned long int features. */ > > +# define ARCH_SHSTK_ENABLE 0x5001 > > +/* Disable SHSTK features in unsigned long int features. */ > > +# define ARCH_SHSTK_DISABLE 0x5002 > > +/* Lock SHSTK features in unsigned long int features. */ > > +# define ARCH_SHSTK_LOCK 0x5003 > > +/* Unlock SHSTK features in unsigned long int features. */ > > +# define ARCH_SHSTK_UNLOCK 0x5004 > > +/* Return SHSTK features in unsigned long int features. */ > > +# define ARCH_SHSTK_STATUS 0x5005 > > + > > +/* ARCH_SHSTK_ features bits */ > > +# define ARCH_SHSTK_SHSTK 0x1 > > +# define ARCH_SHSTK_WRSS 0x2 > > +#endif > > diff --git a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > > index 837a9fd0eb..2ea66c803b 100644 > > --- a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > > +++ b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > > @@ -87,15 +87,14 @@ do_test (void) > > ctx[4].uc_link = &ctx[0]; > > makecontext (&ctx[4], (void (*) (void)) f1, 0); > > > > - /* NB: When shadow stack is enabled, makecontext calls arch_prctl > > - with ARCH_CET_ALLOC_SHSTK to allocate a new shadow stack which > > - can be unmapped. The base address and size of the new shadow > > - stack are returned in __ssp[1] and __ssp[2]. makecontext is > > - called for CTX1, CTX3 and CTX4. But only CTX1 is used. New > > - shadow stacks are allocated in the order of CTX3, CTX1, CTX4. > > - It is very likely that CTX1's shadow stack is placed between > > - CTX3 and CTX4. We munmap CTX3's and CTX4's shadow stacks to > > - create gaps above and below CTX1's shadow stack. We check > > + /* NB: When shadow stack is enabled, makecontext calls map_shadow_stack > > + to allocate a new shadow stack which can be unmapped. The base > > + address and size of the new shadow stack are returned in __ssp[1] > > + and __ssp[2]. makecontext is called for CTX1, CTX3 and CTX4. But > > + only CTX1 is used. New shadow stacks are allocated in the order > > + of CTX3, CTX1, CTX4. It is very likely that CTX1's shadow stack is > > + placed between CTX3 and CTX4. We munmap CTX3's and CTX4's shadow > > + stacks to create gaps above and below CTX1's shadow stack. We check > > that setcontext CTX1 works correctly in this case. */ > > if (_get_ssp () != 0) > > { > > diff --git a/sysdeps/unix/sysv/linux/x86_64/Makefile b/sysdeps/unix/sysv/linux/x86_64/Makefile > > index 5e19202ebf..06b873949e 100644 > > --- a/sysdeps/unix/sysv/linux/x86_64/Makefile > > +++ b/sysdeps/unix/sysv/linux/x86_64/Makefile > > @@ -3,7 +3,7 @@ sysdep_routines += ioperm iopl > > endif > > > > ifeq ($(subdir),stdlib) > > -sysdep_routines += __start_context > > +sysdep_routines += __start_context allocate-shadow-stack > > endif > > > > ifeq ($(subdir),csu) > > diff --git a/sysdeps/unix/sysv/linux/x86_64/__start_context.S b/sysdeps/unix/sysv/linux/x86_64/__start_context.S > > index f6436dd6bb..ae04203c90 100644 > > --- a/sysdeps/unix/sysv/linux/x86_64/__start_context.S > > +++ b/sysdeps/unix/sysv/linux/x86_64/__start_context.S > > @@ -24,20 +24,14 @@ > > /* Use CALL to push __start_context onto the new stack as well as the new > > shadow stack. RDI points to ucontext: > > Incoming: > > - __ssp[0]: The original caller's shadow stack pointer. > > - __ssp[1]: The size of the new shadow stack. > > - __ssp[2]: The size of the new shadow stack. > > - Outgoing: > > __ssp[0]: The new shadow stack pointer. > > __ssp[1]: The base address of the new shadow stack. > > __ssp[2]: The size of the new shadow stack. > > */ > > > > ENTRY(__push___start_context) > > - /* Save the pointer to ucontext. */ > > - movq %rdi, %r9 > > /* Get the original shadow stack pointer. */ > > - rdsspq %r8 > > + rdsspq %rcx > > /* Save the original stack pointer. */ > > movq %rsp, %rdx > > /* Load the top of the new stack into RSI. */ > > @@ -45,24 +39,12 @@ ENTRY(__push___start_context) > > /* Add 8 bytes to RSI since CALL will push the 8-byte return > > address onto stack. */ > > leaq 8(%rsi), %rsp > > - /* Allocate the new shadow stack. The size of the new shadow > > - stack is passed in __ssp[1]. */ > > - lea (oSSP + 8)(%rdi), %RSI_LP > > - movl $ARCH_CET_ALLOC_SHSTK, %edi > > - movl $__NR_arch_prctl, %eax > > - /* The new shadow stack base is returned in __ssp[1]. */ > > - syscall > > - testq %rax, %rax > > - jne L(hlt) /* This should never happen. */ > > - > > - /* Get the size of the new shadow stack. */ > > - movq 8(%rsi), %rdi > > - > > - /* Get the base address of the new shadow stack. */ > > - movq (%rsi), %rsi > > - > > + /* The size of the new shadow stack is stored in __ssp[2]. */ > > + mov (oSSP + 16)(%rdi), %RSI_LP > > + /* The new shadow stack base is stored in __ssp[1]. */ > > + mov (oSSP + 8)(%rdi), %RAX_LP > > /* Use the restore stoken to restore the new shadow stack. */ > > - rstorssp -8(%rsi, %rdi) > > + rstorssp -8(%rax, %rsi) > > > > /* Save the restore token on the original shadow stack. */ > > saveprevssp > > @@ -73,18 +55,12 @@ ENTRY(__push___start_context) > > jmp __start_context > > 1: > > > > - /* Get the new shadow stack pointer. */ > > - rdsspq %rdi > > - > > /* Use the restore stoken to restore the original shadow stack. */ > > - rstorssp -8(%r8) > > + rstorssp -8(%rcx) > > > > /* Save the restore token on the new shadow stack. */ > > saveprevssp > > > > - /* Store the new shadow stack pointer in __ssp[0]. */ > > - movq %rdi, oSSP(%r9) > > - > > /* Restore the original stack. */ > > mov %rdx, %rsp > > ret > > diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > > new file mode 100644 > > index 0000000000..f2e1d03b96 > > --- /dev/null > > +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > > @@ -0,0 +1,55 @@ > > +/* Helper function to allocate shadow stack. > > + Copyright (C) 2023 Free Software Foundation, Inc. > > + This file is part of the GNU C Library. > > + > > + The GNU C Library is free software; you can redistribute it and/or > > + modify it under the terms of the GNU Lesser General Public > > + License as published by the Free Software Foundation; either > > + version 2.1 of the License, or (at your option) any later version. > > + > > + The GNU C Library is distributed in the hope that it will be useful, > > + but WITHOUT ANY WARRANTY; without even the implied warranty of > > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + Lesser General Public License for more details. > > + > > + You should have received a copy of the GNU Lesser General Public > > + License along with the GNU C Library; if not, see > > + <https://www.gnu.org/licenses/>. */ > > + > > +#include <sysdep.h> > > +#include <stdint.h> > > +#include <errno.h> > > +#include <sys/mman.h> > > +#include <libc-pointer-arith.h> > > +#include <allocate-shadow-stack.h> > > + > > +/* NB: This can be treated as a syscall by caller. */ > > + > > +long int > > +__allocate_shadow_stack (size_t stack_size, > > + shadow_stack_size_t *child_stack) > > +{ > > +#ifdef __NR_map_shadow_stack > > + size_t shadow_stack_size > > + = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT; > > + /* Align shadow stack to 8 bytes. */ > > + shadow_stack_size = ALIGN_UP (shadow_stack_size, 8); > > + /* Since sigaltstack shares shadow stack with the current context in > > + the thread, add extra 20 stack frames in shadow stack for signal > > + handlers. */ > > + shadow_stack_size += 20 * 8; > > + void *shadow_stack = (void *)INLINE_SYSCALL_CALL > > + (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN); > > + /* Report the map_shadow_stack error. */ > > + if (shadow_stack == MAP_FAILED) > > + return -errno; > > + > > + /* Save the shadow stack base and size on child stack. */ > > + child_stack[0] = (uintptr_t) shadow_stack; > > + child_stack[1] = shadow_stack_size; > > + > > + return 0; > > +#else > > + return -ENOSYS; > > +#endif > > +} > > diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > > new file mode 100644 > > index 0000000000..d05aaf16e5 > > --- /dev/null > > +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > > @@ -0,0 +1,24 @@ > > +/* Helper function to allocate shadow stack. > > + Copyright (C) 2023 Free Software Foundation, Inc. > > + This file is part of the GNU C Library. > > + > > + The GNU C Library is free software; you can redistribute it and/or > > + modify it under the terms of the GNU Lesser General Public > > + License as published by the Free Software Foundation; either > > + version 2.1 of the License, or (at your option) any later version. > > + > > + The GNU C Library is distributed in the hope that it will be useful, > > + but WITHOUT ANY WARRANTY; without even the implied warranty of > > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > + Lesser General Public License for more details. > > + > > + You should have received a copy of the GNU Lesser General Public > > + License along with the GNU C Library; if not, see > > + <https://www.gnu.org/licenses/>. */ > > + > > +#include <ucontext.h> > > + > > +typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t; > > + > > +extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *) > > + attribute_hidden; > > diff --git a/sysdeps/unix/sysv/linux/x86_64/getcontext.S b/sysdeps/unix/sysv/linux/x86_64/getcontext.S > > index a00e2f6290..71f3802dca 100644 > > --- a/sysdeps/unix/sysv/linux/x86_64/getcontext.S > > +++ b/sysdeps/unix/sysv/linux/x86_64/getcontext.S > > @@ -58,35 +58,15 @@ ENTRY(__getcontext) > > testl $X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET > > jz L(no_shstk) > > > > - /* Save RDI in RDX which won't be clobbered by syscall. */ > > - movq %rdi, %rdx > > - > > xorl %eax, %eax > > cmpq %fs:SSP_BASE_OFFSET, %rax > > jnz L(shadow_stack_bound_recorded) > > > > - /* Get the base address and size of the default shadow stack > > - which must be the current shadow stack since nothing has > > - been recorded yet. */ > > - sub $24, %RSP_LP > > - mov %RSP_LP, %RSI_LP > > - movl $ARCH_CET_STATUS, %edi > > - movl $__NR_arch_prctl, %eax > > - syscall > > - testq %rax, %rax > > - jz L(continue_no_err) > > - > > - /* This should never happen. */ > > - hlt > > - > > -L(continue_no_err): > > - /* Record the base of the current shadow stack. */ > > - movq 8(%rsp), %rax > > + /* When the shadow stack base is unset, the default shadow > > + stack is in use. Use the current shadow stack pointer > > + as the marker for the default shadow stack. */ > > + rdsspq %rax > > movq %rax, %fs:SSP_BASE_OFFSET > > - add $24, %RSP_LP > > - > > - /* Restore RDI. */ > > - movq %rdx, %rdi > > > > L(shadow_stack_bound_recorded): > > /* Get the current shadow stack pointer. */ > > @@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded): > > /* NB: Save the caller's shadow stack so that we can jump back > > to the caller directly. */ > > addq $8, %rax > > - movq %rax, oSSP(%rdx) > > + movq %rax, oSSP(%rdi) > > > > /* Save the current shadow stack base in ucontext. */ > > movq %fs:SSP_BASE_OFFSET, %rax > > diff --git a/sysdeps/unix/sysv/linux/x86_64/makecontext.c b/sysdeps/unix/sysv/linux/x86_64/makecontext.c > > index de9e03eb81..e4f025bd50 100644 > > --- a/sysdeps/unix/sysv/linux/x86_64/makecontext.c > > +++ b/sysdeps/unix/sysv/linux/x86_64/makecontext.c > > @@ -24,6 +24,7 @@ > > # include <pthread.h> > > # include <libc-pointer-arith.h> > > # include <sys/prctl.h> > > +# include <allocate-shadow-stack.h> > > #endif > > > > #include "ucontext_i.h" > > @@ -88,23 +89,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...) > > if ((feature_1 & X86_FEATURE_1_SHSTK) != 0) > > { > > /* Shadow stack is enabled. We need to allocate a new shadow > > - stack. */ > > - unsigned long ssp_size = (((uintptr_t) sp > > - - (uintptr_t) ucp->uc_stack.ss_sp) > > - >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT); > > - /* Align shadow stack to 8 bytes. */ > > - ssp_size = ALIGN_UP (ssp_size, 8); > > - > > - ucp->__ssp[1] = ssp_size; > > - ucp->__ssp[2] = ssp_size; > > - > > - /* Call __push___start_context to allocate a new shadow stack, > > - push __start_context onto the new stack as well as the new > > - shadow stack. NB: After __push___start_context returns, > > + stack. NB: > > ucp->__ssp[0]: The new shadow stack pointer. > > ucp->__ssp[1]: The base address of the new shadow stack. > > ucp->__ssp[2]: The size of the new shadow stack. > > */ > > + long int ret > > + = __allocate_shadow_stack (((uintptr_t) sp > > + - (uintptr_t) ucp->uc_stack.ss_sp), > > + &ucp->__ssp[1]); > > + if (ret != 0) > > + { > > + /* FIXME: What should we do? */ > > + abort (); > > + } > > + > > + ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8; > > + /* Call __push___start_context to push __start_context onto the new > > + stack as well as the new shadow stack. */ > > __push___start_context (ucp); > > } > > else > > diff --git a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > > index 5925752164..2f2fe9875b 100644 > > --- a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > > +++ b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > > @@ -109,25 +109,11 @@ ENTRY(__swapcontext) > > cmpq %fs:SSP_BASE_OFFSET, %rax > > jnz L(shadow_stack_bound_recorded) > > > > - /* Get the base address and size of the default shadow stack > > - which must be the current shadow stack since nothing has > > - been recorded yet. */ > > - sub $24, %RSP_LP > > - mov %RSP_LP, %RSI_LP > > - movl $ARCH_CET_STATUS, %edi > > - movl $__NR_arch_prctl, %eax > > - syscall > > - testq %rax, %rax > > - jz L(continue_no_err) > > - > > - /* This should never happen. */ > > - hlt > > - > > -L(continue_no_err): > > - /* Record the base of the current shadow stack. */ > > - movq 8(%rsp), %rax > > + /* When the shadow stack base is unset, the default shadow > > + stack is in use. Use the current shadow stack pointer > > + as the marker for the default shadow stack. */ > > + rdsspq %rax > > movq %rax, %fs:SSP_BASE_OFFSET > > - add $24, %RSP_LP > > > > L(shadow_stack_bound_recorded): > > /* If we unwind the stack, we can't undo stack unwinding. Just > > diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c > > index 0bf923d48b..f180f0d9a4 100644 > > --- a/sysdeps/x86/cpu-features.c > > +++ b/sysdeps/x86/cpu-features.c > > @@ -1121,8 +1121,9 @@ no_cpuid: > > > > # ifndef SHARED > > /* Check if IBT and SHSTK are enabled by kernel. */ > > - if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT) > > - || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK)) > > + if ((cet_status > > + & (GNU_PROPERTY_X86_FEATURE_1_IBT > > + | GNU_PROPERTY_X86_FEATURE_1_SHSTK))) > > I think the code here and elsewhere would be simplifiable with a > define/enum of > `GNU_PROPERTY_X86_FEATURE_1_SHSTK_OR_IBT = > GNU_PROPERTY_X86_FEATURE_1_SHSTK | GNU_PROPERTY_X86_FEATURE_1_IBT` That may touch more places beyond this patch series. A separate patch after this series has been merged? > > { > > /* Disable IBT and/or SHSTK if they are enabled by kernel, but > > disabled by environment variable: > > @@ -1131,9 +1132,11 @@ no_cpuid: > > */ > > unsigned int cet_feature = 0; > > if (!CPU_FEATURE_USABLE (IBT)) > > - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT; > > + cet_feature |= (cet_status > > + & GNU_PROPERTY_X86_FEATURE_1_IBT); > > if (!CPU_FEATURE_USABLE (SHSTK)) > > - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; > > + cet_feature |= (cet_status > > + & GNU_PROPERTY_X86_FEATURE_1_SHSTK); > > > > if (cet_feature) > > { > > @@ -1148,7 +1151,9 @@ no_cpuid: > > lock CET if IBT or SHSTK is enabled permissively. */ > > if (GL(dl_x86_feature_control).ibt != cet_permissive > > && GL(dl_x86_feature_control).shstk != cet_permissive) > > - dl_cet_lock_cet (); > > + dl_cet_lock_cet (GL(dl_x86_feature_1) > > + & (GNU_PROPERTY_X86_FEATURE_1_IBT > > + | GNU_PROPERTY_X86_FEATURE_1_SHSTK)); > > } > > # endif > > } > > diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c > > index e486e549be..66a78244d4 100644 > > --- a/sysdeps/x86/dl-cet.c > > +++ b/sysdeps/x86/dl-cet.c > > @@ -202,7 +202,7 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info) > > feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; > > > > if (feature_1_lock != 0 > > - && dl_cet_lock_cet () != 0) > > + && dl_cet_lock_cet (feature_1_lock) != 0) > > _dl_fatal_printf ("%s: can't lock CET\n", info->program); > > } > > > > diff --git a/sysdeps/x86_64/nptl/tls.h b/sysdeps/x86_64/nptl/tls.h > > index 1403f939f7..4bcc2552a1 100644 > > --- a/sysdeps/x86_64/nptl/tls.h > > +++ b/sysdeps/x86_64/nptl/tls.h > > @@ -60,7 +60,7 @@ typedef struct > > void *__private_tm[4]; > > /* GCC split stack support. */ > > void *__private_ss; > > - /* The lowest address of shadow stack, */ > > + /* The marker for the current shadow stack. */ > > unsigned long long int ssp_base; > > /* Must be kept even if it is no longer used by glibc since programs, > > like AddressSanitizer, depend on the size of tcbhead_t. */ > > -- > > 2.43.0 > >
On Tue, Dec 26, 2023 at 9:57 AM H.J. Lu <hjl.tools@gmail.com> wrote: > > On Tue, Dec 26, 2023 at 9:38 AM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > > > On Fri, Dec 22, 2023 at 8:58 AM H.J. Lu <hjl.tools@gmail.com> wrote: > > > > > > Sync with Linux kernel 6.6 shadow stack interface. Since only x86-64 is > > > supported, i386 shadow stack codes are unchanged and CET shouldn't be > > > enabled for i386. > > > > > > 1. When the shadow stack base in TCB is unset, the default shadow stack > > > is in use. Use the current shadow stack pointer as the marker for the > > > default shadow stack. It is used to identify if the current shadow stack > > > is the same as the target shadow stack when switching ucontexts. If yes, > > > INCSSP will be used to unwind shadow stack. Otherwise, shadow stack > > > restore token will be used. > > > 2. Allocate shadow stack with the map_shadow_stack syscall. Since there > > > is no function to explicitly release ucontext, there is no place to > > > release shadow stack allocated by map_shadow_stack in ucontext functions. > > > Such shadow stacks will be leaked. > > > 3. Rename arch_prctl CET commands to ARCH_SHSTK_XXX. > > > 4. Rewrite the CET control functions with the current kernel shadow stack > > > interface. > > > > > > Since CET is no longer enabled by kernel, a separate patch will enable > > > shadow stack during startup. > > > --- > > > sysdeps/unix/sysv/linux/x86/bits/mman.h | 5 ++ > > > sysdeps/unix/sysv/linux/x86/cpu-features.c | 13 +++-- > > > sysdeps/unix/sysv/linux/x86/dl-cet.h | 16 ++++-- > > > .../unix/sysv/linux/x86/include/asm/prctl.h | 37 ++++++------- > > > .../sysv/linux/x86/tst-cet-setcontext-1.c | 17 +++--- > > > sysdeps/unix/sysv/linux/x86_64/Makefile | 2 +- > > > .../unix/sysv/linux/x86_64/__start_context.S | 38 +++---------- > > > .../sysv/linux/x86_64/allocate-shadow-stack.c | 55 +++++++++++++++++++ > > > .../sysv/linux/x86_64/allocate-shadow-stack.h | 24 ++++++++ > > > sysdeps/unix/sysv/linux/x86_64/getcontext.S | 30 ++-------- > > > sysdeps/unix/sysv/linux/x86_64/makecontext.c | 28 +++++----- > > > sysdeps/unix/sysv/linux/x86_64/swapcontext.S | 22 ++------ > > > sysdeps/x86/cpu-features.c | 15 +++-- > > > sysdeps/x86/dl-cet.c | 2 +- > > > sysdeps/x86_64/nptl/tls.h | 2 +- > > > 15 files changed, 173 insertions(+), 133 deletions(-) > > > create mode 100644 sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > > > create mode 100644 sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > > > > > > diff --git a/sysdeps/unix/sysv/linux/x86/bits/mman.h b/sysdeps/unix/sysv/linux/x86/bits/mman.h > > > index 3d356e86a0..232b55a13d 100644 > > > --- a/sysdeps/unix/sysv/linux/x86/bits/mman.h > > > +++ b/sysdeps/unix/sysv/linux/x86/bits/mman.h > > > @@ -27,6 +27,11 @@ > > > #define MAP_32BIT 0x40 /* Only give out 32-bit addresses. */ > > > #define MAP_ABOVE4G 0x80 /* Only map above 4GB. */ > > > > > > +#ifdef __USE_MISC > > > +/* Set up a restore token in the newly allocated shadow stack */ > > > +# define SHADOW_STACK_SET_TOKEN 0x1 > > > +#endif > > > + > > > #include <bits/mman-map-flags-generic.h> > > > > > > /* Include generic Linux declarations. */ > > > diff --git a/sysdeps/unix/sysv/linux/x86/cpu-features.c b/sysdeps/unix/sysv/linux/x86/cpu-features.c > > > index 41e7600668..0e6e2bf855 100644 > > > --- a/sysdeps/unix/sysv/linux/x86/cpu-features.c > > > +++ b/sysdeps/unix/sysv/linux/x86/cpu-features.c > > > @@ -23,10 +23,15 @@ > > > static inline int __attribute__ ((always_inline)) > > > get_cet_status (void) > > > { > > > - unsigned long long cet_status[3]; > > > - if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_STATUS, cet_status) == 0) > > > - return cet_status[0]; > > > - return 0; > > > + unsigned long long kernel_feature; > > > + unsigned int status = 0; > > > + if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS, > > > + &kernel_feature) == 0) > > > + { > > > + if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0) > > > + status = GNU_PROPERTY_X86_FEATURE_1_SHSTK; > > > + } > > > + return status; > > > } > > > > > > # ifndef SHARED > > > diff --git a/sysdeps/unix/sysv/linux/x86/dl-cet.h b/sysdeps/unix/sysv/linux/x86/dl-cet.h > > > index c885bf1323..da220ac627 100644 > > > --- a/sysdeps/unix/sysv/linux/x86/dl-cet.h > > > +++ b/sysdeps/unix/sysv/linux/x86/dl-cet.h > > > @@ -21,12 +21,20 @@ > > > static inline int __attribute__ ((always_inline)) > > > dl_cet_disable_cet (unsigned int cet_feature) > > > { > > > - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_DISABLE, > > > - cet_feature); > > > + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) > > > + return -1; > > > + long long int kernel_feature = ARCH_SHSTK_SHSTK; > > > + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_DISABLE, > > > + kernel_feature); > > > } > > > > > > static inline int __attribute__ ((always_inline)) > > > -dl_cet_lock_cet (void) > > > +dl_cet_lock_cet (unsigned int cet_feature) > > > { > > > - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_LOCK, 0); > > > + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) > > > + return -1; > > > + /* Lock all SHSTK features. */ > > > + long long int kernel_feature = -1; > > > + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK, > > > + kernel_feature); > > > } > > > diff --git a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > > > index 45ad0b052f..2f511321ad 100644 > > > --- a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > > > +++ b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h > > > @@ -4,24 +4,19 @@ > > > > > > #include_next <asm/prctl.h> > > > > > > -#ifndef ARCH_CET_STATUS > > > -/* CET features: > > > - IBT: GNU_PROPERTY_X86_FEATURE_1_IBT > > > - SHSTK: GNU_PROPERTY_X86_FEATURE_1_SHSTK > > > - */ > > > -/* Return CET features in unsigned long long *addr: > > > - features: addr[0]. > > > - shadow stack base address: addr[1]. > > > - shadow stack size: addr[2]. > > > - */ > > > -# define ARCH_CET_STATUS 0x3001 > > > -/* Disable CET features in unsigned int features. */ > > > -# define ARCH_CET_DISABLE 0x3002 > > > -/* Lock all CET features. */ > > > -# define ARCH_CET_LOCK 0x3003 > > > -/* Allocate a new shadow stack with unsigned long long *addr: > > > - IN: requested shadow stack size: *addr. > > > - OUT: allocated shadow stack address: *addr. > > > - */ > > > -# define ARCH_CET_ALLOC_SHSTK 0x3004 > > > -#endif /* ARCH_CET_STATUS */ > > > +#ifndef ARCH_SHSTK_ENABLE > > > +/* Enable SHSTK features in unsigned long int features. */ > > > +# define ARCH_SHSTK_ENABLE 0x5001 > > > +/* Disable SHSTK features in unsigned long int features. */ > > > +# define ARCH_SHSTK_DISABLE 0x5002 > > > +/* Lock SHSTK features in unsigned long int features. */ > > > +# define ARCH_SHSTK_LOCK 0x5003 > > > +/* Unlock SHSTK features in unsigned long int features. */ > > > +# define ARCH_SHSTK_UNLOCK 0x5004 > > > +/* Return SHSTK features in unsigned long int features. */ > > > +# define ARCH_SHSTK_STATUS 0x5005 > > > + > > > +/* ARCH_SHSTK_ features bits */ > > > +# define ARCH_SHSTK_SHSTK 0x1 > > > +# define ARCH_SHSTK_WRSS 0x2 > > > +#endif > > > diff --git a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > > > index 837a9fd0eb..2ea66c803b 100644 > > > --- a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > > > +++ b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c > > > @@ -87,15 +87,14 @@ do_test (void) > > > ctx[4].uc_link = &ctx[0]; > > > makecontext (&ctx[4], (void (*) (void)) f1, 0); > > > > > > - /* NB: When shadow stack is enabled, makecontext calls arch_prctl > > > - with ARCH_CET_ALLOC_SHSTK to allocate a new shadow stack which > > > - can be unmapped. The base address and size of the new shadow > > > - stack are returned in __ssp[1] and __ssp[2]. makecontext is > > > - called for CTX1, CTX3 and CTX4. But only CTX1 is used. New > > > - shadow stacks are allocated in the order of CTX3, CTX1, CTX4. > > > - It is very likely that CTX1's shadow stack is placed between > > > - CTX3 and CTX4. We munmap CTX3's and CTX4's shadow stacks to > > > - create gaps above and below CTX1's shadow stack. We check > > > + /* NB: When shadow stack is enabled, makecontext calls map_shadow_stack > > > + to allocate a new shadow stack which can be unmapped. The base > > > + address and size of the new shadow stack are returned in __ssp[1] > > > + and __ssp[2]. makecontext is called for CTX1, CTX3 and CTX4. But > > > + only CTX1 is used. New shadow stacks are allocated in the order > > > + of CTX3, CTX1, CTX4. It is very likely that CTX1's shadow stack is > > > + placed between CTX3 and CTX4. We munmap CTX3's and CTX4's shadow > > > + stacks to create gaps above and below CTX1's shadow stack. We check > > > that setcontext CTX1 works correctly in this case. */ > > > if (_get_ssp () != 0) > > > { > > > diff --git a/sysdeps/unix/sysv/linux/x86_64/Makefile b/sysdeps/unix/sysv/linux/x86_64/Makefile > > > index 5e19202ebf..06b873949e 100644 > > > --- a/sysdeps/unix/sysv/linux/x86_64/Makefile > > > +++ b/sysdeps/unix/sysv/linux/x86_64/Makefile > > > @@ -3,7 +3,7 @@ sysdep_routines += ioperm iopl > > > endif > > > > > > ifeq ($(subdir),stdlib) > > > -sysdep_routines += __start_context > > > +sysdep_routines += __start_context allocate-shadow-stack > > > endif > > > > > > ifeq ($(subdir),csu) > > > diff --git a/sysdeps/unix/sysv/linux/x86_64/__start_context.S b/sysdeps/unix/sysv/linux/x86_64/__start_context.S > > > index f6436dd6bb..ae04203c90 100644 > > > --- a/sysdeps/unix/sysv/linux/x86_64/__start_context.S > > > +++ b/sysdeps/unix/sysv/linux/x86_64/__start_context.S > > > @@ -24,20 +24,14 @@ > > > /* Use CALL to push __start_context onto the new stack as well as the new > > > shadow stack. RDI points to ucontext: > > > Incoming: > > > - __ssp[0]: The original caller's shadow stack pointer. > > > - __ssp[1]: The size of the new shadow stack. > > > - __ssp[2]: The size of the new shadow stack. > > > - Outgoing: > > > __ssp[0]: The new shadow stack pointer. > > > __ssp[1]: The base address of the new shadow stack. > > > __ssp[2]: The size of the new shadow stack. > > > */ > > > > > > ENTRY(__push___start_context) > > > - /* Save the pointer to ucontext. */ > > > - movq %rdi, %r9 > > > /* Get the original shadow stack pointer. */ > > > - rdsspq %r8 > > > + rdsspq %rcx > > > /* Save the original stack pointer. */ > > > movq %rsp, %rdx > > > /* Load the top of the new stack into RSI. */ > > > @@ -45,24 +39,12 @@ ENTRY(__push___start_context) > > > /* Add 8 bytes to RSI since CALL will push the 8-byte return > > > address onto stack. */ > > > leaq 8(%rsi), %rsp > > > - /* Allocate the new shadow stack. The size of the new shadow > > > - stack is passed in __ssp[1]. */ > > > - lea (oSSP + 8)(%rdi), %RSI_LP > > > - movl $ARCH_CET_ALLOC_SHSTK, %edi > > > - movl $__NR_arch_prctl, %eax > > > - /* The new shadow stack base is returned in __ssp[1]. */ > > > - syscall > > > - testq %rax, %rax > > > - jne L(hlt) /* This should never happen. */ > > > - > > > - /* Get the size of the new shadow stack. */ > > > - movq 8(%rsi), %rdi > > > - > > > - /* Get the base address of the new shadow stack. */ > > > - movq (%rsi), %rsi > > > - > > > + /* The size of the new shadow stack is stored in __ssp[2]. */ > > > + mov (oSSP + 16)(%rdi), %RSI_LP > > > + /* The new shadow stack base is stored in __ssp[1]. */ > > > + mov (oSSP + 8)(%rdi), %RAX_LP > > > /* Use the restore stoken to restore the new shadow stack. */ > > > - rstorssp -8(%rsi, %rdi) > > > + rstorssp -8(%rax, %rsi) > > > > > > /* Save the restore token on the original shadow stack. */ > > > saveprevssp > > > @@ -73,18 +55,12 @@ ENTRY(__push___start_context) > > > jmp __start_context > > > 1: > > > > > > - /* Get the new shadow stack pointer. */ > > > - rdsspq %rdi > > > - > > > /* Use the restore stoken to restore the original shadow stack. */ > > > - rstorssp -8(%r8) > > > + rstorssp -8(%rcx) > > > > > > /* Save the restore token on the new shadow stack. */ > > > saveprevssp > > > > > > - /* Store the new shadow stack pointer in __ssp[0]. */ > > > - movq %rdi, oSSP(%r9) > > > - > > > /* Restore the original stack. */ > > > mov %rdx, %rsp > > > ret > > > diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > > > new file mode 100644 > > > index 0000000000..f2e1d03b96 > > > --- /dev/null > > > +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c > > > @@ -0,0 +1,55 @@ > > > +/* Helper function to allocate shadow stack. > > > + Copyright (C) 2023 Free Software Foundation, Inc. > > > + This file is part of the GNU C Library. > > > + > > > + The GNU C Library is free software; you can redistribute it and/or > > > + modify it under the terms of the GNU Lesser General Public > > > + License as published by the Free Software Foundation; either > > > + version 2.1 of the License, or (at your option) any later version. > > > + > > > + The GNU C Library is distributed in the hope that it will be useful, > > > + but WITHOUT ANY WARRANTY; without even the implied warranty of > > > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > > + Lesser General Public License for more details. > > > + > > > + You should have received a copy of the GNU Lesser General Public > > > + License along with the GNU C Library; if not, see > > > + <https://www.gnu.org/licenses/>. */ > > > + > > > +#include <sysdep.h> > > > +#include <stdint.h> > > > +#include <errno.h> > > > +#include <sys/mman.h> > > > +#include <libc-pointer-arith.h> > > > +#include <allocate-shadow-stack.h> > > > + > > > +/* NB: This can be treated as a syscall by caller. */ > > > + > > > +long int > > > +__allocate_shadow_stack (size_t stack_size, > > > + shadow_stack_size_t *child_stack) > > > +{ > > > +#ifdef __NR_map_shadow_stack > > > + size_t shadow_stack_size > > > + = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT; > > > + /* Align shadow stack to 8 bytes. */ > > > + shadow_stack_size = ALIGN_UP (shadow_stack_size, 8); > > > + /* Since sigaltstack shares shadow stack with the current context in > > > + the thread, add extra 20 stack frames in shadow stack for signal > > > + handlers. */ > > > + shadow_stack_size += 20 * 8; > > > + void *shadow_stack = (void *)INLINE_SYSCALL_CALL > > > + (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN); > > > + /* Report the map_shadow_stack error. */ > > > + if (shadow_stack == MAP_FAILED) > > > + return -errno; > > > + > > > + /* Save the shadow stack base and size on child stack. */ > > > + child_stack[0] = (uintptr_t) shadow_stack; > > > + child_stack[1] = shadow_stack_size; > > > + > > > + return 0; > > > +#else > > > + return -ENOSYS; > > > +#endif > > > +} > > > diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > > > new file mode 100644 > > > index 0000000000..d05aaf16e5 > > > --- /dev/null > > > +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h > > > @@ -0,0 +1,24 @@ > > > +/* Helper function to allocate shadow stack. > > > + Copyright (C) 2023 Free Software Foundation, Inc. > > > + This file is part of the GNU C Library. > > > + > > > + The GNU C Library is free software; you can redistribute it and/or > > > + modify it under the terms of the GNU Lesser General Public > > > + License as published by the Free Software Foundation; either > > > + version 2.1 of the License, or (at your option) any later version. > > > + > > > + The GNU C Library is distributed in the hope that it will be useful, > > > + but WITHOUT ANY WARRANTY; without even the implied warranty of > > > + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > > > + Lesser General Public License for more details. > > > + > > > + You should have received a copy of the GNU Lesser General Public > > > + License along with the GNU C Library; if not, see > > > + <https://www.gnu.org/licenses/>. */ > > > + > > > +#include <ucontext.h> > > > + > > > +typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t; > > > + > > > +extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *) > > > + attribute_hidden; > > > diff --git a/sysdeps/unix/sysv/linux/x86_64/getcontext.S b/sysdeps/unix/sysv/linux/x86_64/getcontext.S > > > index a00e2f6290..71f3802dca 100644 > > > --- a/sysdeps/unix/sysv/linux/x86_64/getcontext.S > > > +++ b/sysdeps/unix/sysv/linux/x86_64/getcontext.S > > > @@ -58,35 +58,15 @@ ENTRY(__getcontext) > > > testl $X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET > > > jz L(no_shstk) > > > > > > - /* Save RDI in RDX which won't be clobbered by syscall. */ > > > - movq %rdi, %rdx > > > - > > > xorl %eax, %eax > > > cmpq %fs:SSP_BASE_OFFSET, %rax > > > jnz L(shadow_stack_bound_recorded) > > > > > > - /* Get the base address and size of the default shadow stack > > > - which must be the current shadow stack since nothing has > > > - been recorded yet. */ > > > - sub $24, %RSP_LP > > > - mov %RSP_LP, %RSI_LP > > > - movl $ARCH_CET_STATUS, %edi > > > - movl $__NR_arch_prctl, %eax > > > - syscall > > > - testq %rax, %rax > > > - jz L(continue_no_err) > > > - > > > - /* This should never happen. */ > > > - hlt > > > - > > > -L(continue_no_err): > > > - /* Record the base of the current shadow stack. */ > > > - movq 8(%rsp), %rax > > > + /* When the shadow stack base is unset, the default shadow > > > + stack is in use. Use the current shadow stack pointer > > > + as the marker for the default shadow stack. */ > > > + rdsspq %rax > > > movq %rax, %fs:SSP_BASE_OFFSET > > > - add $24, %RSP_LP > > > - > > > - /* Restore RDI. */ > > > - movq %rdx, %rdi > > > > > > L(shadow_stack_bound_recorded): > > > /* Get the current shadow stack pointer. */ > > > @@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded): > > > /* NB: Save the caller's shadow stack so that we can jump back > > > to the caller directly. */ > > > addq $8, %rax > > > - movq %rax, oSSP(%rdx) > > > + movq %rax, oSSP(%rdi) > > > > > > /* Save the current shadow stack base in ucontext. */ > > > movq %fs:SSP_BASE_OFFSET, %rax > > > diff --git a/sysdeps/unix/sysv/linux/x86_64/makecontext.c b/sysdeps/unix/sysv/linux/x86_64/makecontext.c > > > index de9e03eb81..e4f025bd50 100644 > > > --- a/sysdeps/unix/sysv/linux/x86_64/makecontext.c > > > +++ b/sysdeps/unix/sysv/linux/x86_64/makecontext.c > > > @@ -24,6 +24,7 @@ > > > # include <pthread.h> > > > # include <libc-pointer-arith.h> > > > # include <sys/prctl.h> > > > +# include <allocate-shadow-stack.h> > > > #endif > > > > > > #include "ucontext_i.h" > > > @@ -88,23 +89,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...) > > > if ((feature_1 & X86_FEATURE_1_SHSTK) != 0) > > > { > > > /* Shadow stack is enabled. We need to allocate a new shadow > > > - stack. */ > > > - unsigned long ssp_size = (((uintptr_t) sp > > > - - (uintptr_t) ucp->uc_stack.ss_sp) > > > - >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT); > > > - /* Align shadow stack to 8 bytes. */ > > > - ssp_size = ALIGN_UP (ssp_size, 8); > > > - > > > - ucp->__ssp[1] = ssp_size; > > > - ucp->__ssp[2] = ssp_size; > > > - > > > - /* Call __push___start_context to allocate a new shadow stack, > > > - push __start_context onto the new stack as well as the new > > > - shadow stack. NB: After __push___start_context returns, > > > + stack. NB: > > > ucp->__ssp[0]: The new shadow stack pointer. > > > ucp->__ssp[1]: The base address of the new shadow stack. > > > ucp->__ssp[2]: The size of the new shadow stack. > > > */ > > > + long int ret > > > + = __allocate_shadow_stack (((uintptr_t) sp > > > + - (uintptr_t) ucp->uc_stack.ss_sp), > > > + &ucp->__ssp[1]); > > > + if (ret != 0) > > > + { > > > + /* FIXME: What should we do? */ > > > + abort (); > > > + } > > > + > > > + ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8; > > > + /* Call __push___start_context to push __start_context onto the new > > > + stack as well as the new shadow stack. */ > > > __push___start_context (ucp); > > > } > > > else > > > diff --git a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > > > index 5925752164..2f2fe9875b 100644 > > > --- a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > > > +++ b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S > > > @@ -109,25 +109,11 @@ ENTRY(__swapcontext) > > > cmpq %fs:SSP_BASE_OFFSET, %rax > > > jnz L(shadow_stack_bound_recorded) > > > > > > - /* Get the base address and size of the default shadow stack > > > - which must be the current shadow stack since nothing has > > > - been recorded yet. */ > > > - sub $24, %RSP_LP > > > - mov %RSP_LP, %RSI_LP > > > - movl $ARCH_CET_STATUS, %edi > > > - movl $__NR_arch_prctl, %eax > > > - syscall > > > - testq %rax, %rax > > > - jz L(continue_no_err) > > > - > > > - /* This should never happen. */ > > > - hlt > > > - > > > -L(continue_no_err): > > > - /* Record the base of the current shadow stack. */ > > > - movq 8(%rsp), %rax > > > + /* When the shadow stack base is unset, the default shadow > > > + stack is in use. Use the current shadow stack pointer > > > + as the marker for the default shadow stack. */ > > > + rdsspq %rax > > > movq %rax, %fs:SSP_BASE_OFFSET > > > - add $24, %RSP_LP > > > > > > L(shadow_stack_bound_recorded): > > > /* If we unwind the stack, we can't undo stack unwinding. Just > > > diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c > > > index 0bf923d48b..f180f0d9a4 100644 > > > --- a/sysdeps/x86/cpu-features.c > > > +++ b/sysdeps/x86/cpu-features.c > > > @@ -1121,8 +1121,9 @@ no_cpuid: > > > > > > # ifndef SHARED > > > /* Check if IBT and SHSTK are enabled by kernel. */ > > > - if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT) > > > - || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK)) > > > + if ((cet_status > > > + & (GNU_PROPERTY_X86_FEATURE_1_IBT > > > + | GNU_PROPERTY_X86_FEATURE_1_SHSTK))) > > > > I think the code here and elsewhere would be simplifiable with a > > define/enum of > > `GNU_PROPERTY_X86_FEATURE_1_SHSTK_OR_IBT = > > GNU_PROPERTY_X86_FEATURE_1_SHSTK | GNU_PROPERTY_X86_FEATURE_1_IBT` > > That may touch more places beyond this patch series. A separate > patch after this series has been merged? Fair enough. > > > > { > > > /* Disable IBT and/or SHSTK if they are enabled by kernel, but > > > disabled by environment variable: > > > @@ -1131,9 +1132,11 @@ no_cpuid: > > > */ > > > unsigned int cet_feature = 0; > > > if (!CPU_FEATURE_USABLE (IBT)) > > > - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT; > > > + cet_feature |= (cet_status > > > + & GNU_PROPERTY_X86_FEATURE_1_IBT); > > > if (!CPU_FEATURE_USABLE (SHSTK)) > > > - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; > > > + cet_feature |= (cet_status > > > + & GNU_PROPERTY_X86_FEATURE_1_SHSTK); > > > > > > if (cet_feature) > > > { > > > @@ -1148,7 +1151,9 @@ no_cpuid: > > > lock CET if IBT or SHSTK is enabled permissively. */ > > > if (GL(dl_x86_feature_control).ibt != cet_permissive > > > && GL(dl_x86_feature_control).shstk != cet_permissive) > > > - dl_cet_lock_cet (); > > > + dl_cet_lock_cet (GL(dl_x86_feature_1) > > > + & (GNU_PROPERTY_X86_FEATURE_1_IBT > > > + | GNU_PROPERTY_X86_FEATURE_1_SHSTK)); > > > } > > > # endif > > > } > > > diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c > > > index e486e549be..66a78244d4 100644 > > > --- a/sysdeps/x86/dl-cet.c > > > +++ b/sysdeps/x86/dl-cet.c > > > @@ -202,7 +202,7 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info) > > > feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; > > > > > > if (feature_1_lock != 0 > > > - && dl_cet_lock_cet () != 0) > > > + && dl_cet_lock_cet (feature_1_lock) != 0) > > > _dl_fatal_printf ("%s: can't lock CET\n", info->program); > > > } > > > > > > diff --git a/sysdeps/x86_64/nptl/tls.h b/sysdeps/x86_64/nptl/tls.h > > > index 1403f939f7..4bcc2552a1 100644 > > > --- a/sysdeps/x86_64/nptl/tls.h > > > +++ b/sysdeps/x86_64/nptl/tls.h > > > @@ -60,7 +60,7 @@ typedef struct > > > void *__private_tm[4]; > > > /* GCC split stack support. */ > > > void *__private_ss; > > > - /* The lowest address of shadow stack, */ > > > + /* The marker for the current shadow stack. */ > > > unsigned long long int ssp_base; > > > /* Must be kept even if it is no longer used by glibc since programs, > > > like AddressSanitizer, depend on the size of tcbhead_t. */ > > > -- > > > 2.43.0 > > > > > > > -- > H.J.
diff --git a/sysdeps/unix/sysv/linux/x86/bits/mman.h b/sysdeps/unix/sysv/linux/x86/bits/mman.h index 3d356e86a0..232b55a13d 100644 --- a/sysdeps/unix/sysv/linux/x86/bits/mman.h +++ b/sysdeps/unix/sysv/linux/x86/bits/mman.h @@ -27,6 +27,11 @@ #define MAP_32BIT 0x40 /* Only give out 32-bit addresses. */ #define MAP_ABOVE4G 0x80 /* Only map above 4GB. */ +#ifdef __USE_MISC +/* Set up a restore token in the newly allocated shadow stack */ +# define SHADOW_STACK_SET_TOKEN 0x1 +#endif + #include <bits/mman-map-flags-generic.h> /* Include generic Linux declarations. */ diff --git a/sysdeps/unix/sysv/linux/x86/cpu-features.c b/sysdeps/unix/sysv/linux/x86/cpu-features.c index 41e7600668..0e6e2bf855 100644 --- a/sysdeps/unix/sysv/linux/x86/cpu-features.c +++ b/sysdeps/unix/sysv/linux/x86/cpu-features.c @@ -23,10 +23,15 @@ static inline int __attribute__ ((always_inline)) get_cet_status (void) { - unsigned long long cet_status[3]; - if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_STATUS, cet_status) == 0) - return cet_status[0]; - return 0; + unsigned long long kernel_feature; + unsigned int status = 0; + if (INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_STATUS, + &kernel_feature) == 0) + { + if ((kernel_feature & ARCH_SHSTK_SHSTK) != 0) + status = GNU_PROPERTY_X86_FEATURE_1_SHSTK; + } + return status; } # ifndef SHARED diff --git a/sysdeps/unix/sysv/linux/x86/dl-cet.h b/sysdeps/unix/sysv/linux/x86/dl-cet.h index c885bf1323..da220ac627 100644 --- a/sysdeps/unix/sysv/linux/x86/dl-cet.h +++ b/sysdeps/unix/sysv/linux/x86/dl-cet.h @@ -21,12 +21,20 @@ static inline int __attribute__ ((always_inline)) dl_cet_disable_cet (unsigned int cet_feature) { - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_DISABLE, - cet_feature); + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) + return -1; + long long int kernel_feature = ARCH_SHSTK_SHSTK; + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_DISABLE, + kernel_feature); } static inline int __attribute__ ((always_inline)) -dl_cet_lock_cet (void) +dl_cet_lock_cet (unsigned int cet_feature) { - return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_CET_LOCK, 0); + if (cet_feature != GNU_PROPERTY_X86_FEATURE_1_SHSTK) + return -1; + /* Lock all SHSTK features. */ + long long int kernel_feature = -1; + return (int) INTERNAL_SYSCALL_CALL (arch_prctl, ARCH_SHSTK_LOCK, + kernel_feature); } diff --git a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h index 45ad0b052f..2f511321ad 100644 --- a/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h +++ b/sysdeps/unix/sysv/linux/x86/include/asm/prctl.h @@ -4,24 +4,19 @@ #include_next <asm/prctl.h> -#ifndef ARCH_CET_STATUS -/* CET features: - IBT: GNU_PROPERTY_X86_FEATURE_1_IBT - SHSTK: GNU_PROPERTY_X86_FEATURE_1_SHSTK - */ -/* Return CET features in unsigned long long *addr: - features: addr[0]. - shadow stack base address: addr[1]. - shadow stack size: addr[2]. - */ -# define ARCH_CET_STATUS 0x3001 -/* Disable CET features in unsigned int features. */ -# define ARCH_CET_DISABLE 0x3002 -/* Lock all CET features. */ -# define ARCH_CET_LOCK 0x3003 -/* Allocate a new shadow stack with unsigned long long *addr: - IN: requested shadow stack size: *addr. - OUT: allocated shadow stack address: *addr. - */ -# define ARCH_CET_ALLOC_SHSTK 0x3004 -#endif /* ARCH_CET_STATUS */ +#ifndef ARCH_SHSTK_ENABLE +/* Enable SHSTK features in unsigned long int features. */ +# define ARCH_SHSTK_ENABLE 0x5001 +/* Disable SHSTK features in unsigned long int features. */ +# define ARCH_SHSTK_DISABLE 0x5002 +/* Lock SHSTK features in unsigned long int features. */ +# define ARCH_SHSTK_LOCK 0x5003 +/* Unlock SHSTK features in unsigned long int features. */ +# define ARCH_SHSTK_UNLOCK 0x5004 +/* Return SHSTK features in unsigned long int features. */ +# define ARCH_SHSTK_STATUS 0x5005 + +/* ARCH_SHSTK_ features bits */ +# define ARCH_SHSTK_SHSTK 0x1 +# define ARCH_SHSTK_WRSS 0x2 +#endif diff --git a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c index 837a9fd0eb..2ea66c803b 100644 --- a/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c +++ b/sysdeps/unix/sysv/linux/x86/tst-cet-setcontext-1.c @@ -87,15 +87,14 @@ do_test (void) ctx[4].uc_link = &ctx[0]; makecontext (&ctx[4], (void (*) (void)) f1, 0); - /* NB: When shadow stack is enabled, makecontext calls arch_prctl - with ARCH_CET_ALLOC_SHSTK to allocate a new shadow stack which - can be unmapped. The base address and size of the new shadow - stack are returned in __ssp[1] and __ssp[2]. makecontext is - called for CTX1, CTX3 and CTX4. But only CTX1 is used. New - shadow stacks are allocated in the order of CTX3, CTX1, CTX4. - It is very likely that CTX1's shadow stack is placed between - CTX3 and CTX4. We munmap CTX3's and CTX4's shadow stacks to - create gaps above and below CTX1's shadow stack. We check + /* NB: When shadow stack is enabled, makecontext calls map_shadow_stack + to allocate a new shadow stack which can be unmapped. The base + address and size of the new shadow stack are returned in __ssp[1] + and __ssp[2]. makecontext is called for CTX1, CTX3 and CTX4. But + only CTX1 is used. New shadow stacks are allocated in the order + of CTX3, CTX1, CTX4. It is very likely that CTX1's shadow stack is + placed between CTX3 and CTX4. We munmap CTX3's and CTX4's shadow + stacks to create gaps above and below CTX1's shadow stack. We check that setcontext CTX1 works correctly in this case. */ if (_get_ssp () != 0) { diff --git a/sysdeps/unix/sysv/linux/x86_64/Makefile b/sysdeps/unix/sysv/linux/x86_64/Makefile index 5e19202ebf..06b873949e 100644 --- a/sysdeps/unix/sysv/linux/x86_64/Makefile +++ b/sysdeps/unix/sysv/linux/x86_64/Makefile @@ -3,7 +3,7 @@ sysdep_routines += ioperm iopl endif ifeq ($(subdir),stdlib) -sysdep_routines += __start_context +sysdep_routines += __start_context allocate-shadow-stack endif ifeq ($(subdir),csu) diff --git a/sysdeps/unix/sysv/linux/x86_64/__start_context.S b/sysdeps/unix/sysv/linux/x86_64/__start_context.S index f6436dd6bb..ae04203c90 100644 --- a/sysdeps/unix/sysv/linux/x86_64/__start_context.S +++ b/sysdeps/unix/sysv/linux/x86_64/__start_context.S @@ -24,20 +24,14 @@ /* Use CALL to push __start_context onto the new stack as well as the new shadow stack. RDI points to ucontext: Incoming: - __ssp[0]: The original caller's shadow stack pointer. - __ssp[1]: The size of the new shadow stack. - __ssp[2]: The size of the new shadow stack. - Outgoing: __ssp[0]: The new shadow stack pointer. __ssp[1]: The base address of the new shadow stack. __ssp[2]: The size of the new shadow stack. */ ENTRY(__push___start_context) - /* Save the pointer to ucontext. */ - movq %rdi, %r9 /* Get the original shadow stack pointer. */ - rdsspq %r8 + rdsspq %rcx /* Save the original stack pointer. */ movq %rsp, %rdx /* Load the top of the new stack into RSI. */ @@ -45,24 +39,12 @@ ENTRY(__push___start_context) /* Add 8 bytes to RSI since CALL will push the 8-byte return address onto stack. */ leaq 8(%rsi), %rsp - /* Allocate the new shadow stack. The size of the new shadow - stack is passed in __ssp[1]. */ - lea (oSSP + 8)(%rdi), %RSI_LP - movl $ARCH_CET_ALLOC_SHSTK, %edi - movl $__NR_arch_prctl, %eax - /* The new shadow stack base is returned in __ssp[1]. */ - syscall - testq %rax, %rax - jne L(hlt) /* This should never happen. */ - - /* Get the size of the new shadow stack. */ - movq 8(%rsi), %rdi - - /* Get the base address of the new shadow stack. */ - movq (%rsi), %rsi - + /* The size of the new shadow stack is stored in __ssp[2]. */ + mov (oSSP + 16)(%rdi), %RSI_LP + /* The new shadow stack base is stored in __ssp[1]. */ + mov (oSSP + 8)(%rdi), %RAX_LP /* Use the restore stoken to restore the new shadow stack. */ - rstorssp -8(%rsi, %rdi) + rstorssp -8(%rax, %rsi) /* Save the restore token on the original shadow stack. */ saveprevssp @@ -73,18 +55,12 @@ ENTRY(__push___start_context) jmp __start_context 1: - /* Get the new shadow stack pointer. */ - rdsspq %rdi - /* Use the restore stoken to restore the original shadow stack. */ - rstorssp -8(%r8) + rstorssp -8(%rcx) /* Save the restore token on the new shadow stack. */ saveprevssp - /* Store the new shadow stack pointer in __ssp[0]. */ - movq %rdi, oSSP(%r9) - /* Restore the original stack. */ mov %rdx, %rsp ret diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c new file mode 100644 index 0000000000..f2e1d03b96 --- /dev/null +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.c @@ -0,0 +1,55 @@ +/* Helper function to allocate shadow stack. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <sysdep.h> +#include <stdint.h> +#include <errno.h> +#include <sys/mman.h> +#include <libc-pointer-arith.h> +#include <allocate-shadow-stack.h> + +/* NB: This can be treated as a syscall by caller. */ + +long int +__allocate_shadow_stack (size_t stack_size, + shadow_stack_size_t *child_stack) +{ +#ifdef __NR_map_shadow_stack + size_t shadow_stack_size + = stack_size >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT; + /* Align shadow stack to 8 bytes. */ + shadow_stack_size = ALIGN_UP (shadow_stack_size, 8); + /* Since sigaltstack shares shadow stack with the current context in + the thread, add extra 20 stack frames in shadow stack for signal + handlers. */ + shadow_stack_size += 20 * 8; + void *shadow_stack = (void *)INLINE_SYSCALL_CALL + (map_shadow_stack, NULL, shadow_stack_size, SHADOW_STACK_SET_TOKEN); + /* Report the map_shadow_stack error. */ + if (shadow_stack == MAP_FAILED) + return -errno; + + /* Save the shadow stack base and size on child stack. */ + child_stack[0] = (uintptr_t) shadow_stack; + child_stack[1] = shadow_stack_size; + + return 0; +#else + return -ENOSYS; +#endif +} diff --git a/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h new file mode 100644 index 0000000000..d05aaf16e5 --- /dev/null +++ b/sysdeps/unix/sysv/linux/x86_64/allocate-shadow-stack.h @@ -0,0 +1,24 @@ +/* Helper function to allocate shadow stack. + Copyright (C) 2023 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <ucontext.h> + +typedef __typeof (((ucontext_t *) 0)->__ssp[0]) shadow_stack_size_t; + +extern long int __allocate_shadow_stack (size_t, shadow_stack_size_t *) + attribute_hidden; diff --git a/sysdeps/unix/sysv/linux/x86_64/getcontext.S b/sysdeps/unix/sysv/linux/x86_64/getcontext.S index a00e2f6290..71f3802dca 100644 --- a/sysdeps/unix/sysv/linux/x86_64/getcontext.S +++ b/sysdeps/unix/sysv/linux/x86_64/getcontext.S @@ -58,35 +58,15 @@ ENTRY(__getcontext) testl $X86_FEATURE_1_SHSTK, %fs:FEATURE_1_OFFSET jz L(no_shstk) - /* Save RDI in RDX which won't be clobbered by syscall. */ - movq %rdi, %rdx - xorl %eax, %eax cmpq %fs:SSP_BASE_OFFSET, %rax jnz L(shadow_stack_bound_recorded) - /* Get the base address and size of the default shadow stack - which must be the current shadow stack since nothing has - been recorded yet. */ - sub $24, %RSP_LP - mov %RSP_LP, %RSI_LP - movl $ARCH_CET_STATUS, %edi - movl $__NR_arch_prctl, %eax - syscall - testq %rax, %rax - jz L(continue_no_err) - - /* This should never happen. */ - hlt - -L(continue_no_err): - /* Record the base of the current shadow stack. */ - movq 8(%rsp), %rax + /* When the shadow stack base is unset, the default shadow + stack is in use. Use the current shadow stack pointer + as the marker for the default shadow stack. */ + rdsspq %rax movq %rax, %fs:SSP_BASE_OFFSET - add $24, %RSP_LP - - /* Restore RDI. */ - movq %rdx, %rdi L(shadow_stack_bound_recorded): /* Get the current shadow stack pointer. */ @@ -94,7 +74,7 @@ L(shadow_stack_bound_recorded): /* NB: Save the caller's shadow stack so that we can jump back to the caller directly. */ addq $8, %rax - movq %rax, oSSP(%rdx) + movq %rax, oSSP(%rdi) /* Save the current shadow stack base in ucontext. */ movq %fs:SSP_BASE_OFFSET, %rax diff --git a/sysdeps/unix/sysv/linux/x86_64/makecontext.c b/sysdeps/unix/sysv/linux/x86_64/makecontext.c index de9e03eb81..e4f025bd50 100644 --- a/sysdeps/unix/sysv/linux/x86_64/makecontext.c +++ b/sysdeps/unix/sysv/linux/x86_64/makecontext.c @@ -24,6 +24,7 @@ # include <pthread.h> # include <libc-pointer-arith.h> # include <sys/prctl.h> +# include <allocate-shadow-stack.h> #endif #include "ucontext_i.h" @@ -88,23 +89,24 @@ __makecontext (ucontext_t *ucp, void (*func) (void), int argc, ...) if ((feature_1 & X86_FEATURE_1_SHSTK) != 0) { /* Shadow stack is enabled. We need to allocate a new shadow - stack. */ - unsigned long ssp_size = (((uintptr_t) sp - - (uintptr_t) ucp->uc_stack.ss_sp) - >> STACK_SIZE_TO_SHADOW_STACK_SIZE_SHIFT); - /* Align shadow stack to 8 bytes. */ - ssp_size = ALIGN_UP (ssp_size, 8); - - ucp->__ssp[1] = ssp_size; - ucp->__ssp[2] = ssp_size; - - /* Call __push___start_context to allocate a new shadow stack, - push __start_context onto the new stack as well as the new - shadow stack. NB: After __push___start_context returns, + stack. NB: ucp->__ssp[0]: The new shadow stack pointer. ucp->__ssp[1]: The base address of the new shadow stack. ucp->__ssp[2]: The size of the new shadow stack. */ + long int ret + = __allocate_shadow_stack (((uintptr_t) sp + - (uintptr_t) ucp->uc_stack.ss_sp), + &ucp->__ssp[1]); + if (ret != 0) + { + /* FIXME: What should we do? */ + abort (); + } + + ucp->__ssp[0] = ucp->__ssp[1] + ucp->__ssp[2] - 8; + /* Call __push___start_context to push __start_context onto the new + stack as well as the new shadow stack. */ __push___start_context (ucp); } else diff --git a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S index 5925752164..2f2fe9875b 100644 --- a/sysdeps/unix/sysv/linux/x86_64/swapcontext.S +++ b/sysdeps/unix/sysv/linux/x86_64/swapcontext.S @@ -109,25 +109,11 @@ ENTRY(__swapcontext) cmpq %fs:SSP_BASE_OFFSET, %rax jnz L(shadow_stack_bound_recorded) - /* Get the base address and size of the default shadow stack - which must be the current shadow stack since nothing has - been recorded yet. */ - sub $24, %RSP_LP - mov %RSP_LP, %RSI_LP - movl $ARCH_CET_STATUS, %edi - movl $__NR_arch_prctl, %eax - syscall - testq %rax, %rax - jz L(continue_no_err) - - /* This should never happen. */ - hlt - -L(continue_no_err): - /* Record the base of the current shadow stack. */ - movq 8(%rsp), %rax + /* When the shadow stack base is unset, the default shadow + stack is in use. Use the current shadow stack pointer + as the marker for the default shadow stack. */ + rdsspq %rax movq %rax, %fs:SSP_BASE_OFFSET - add $24, %RSP_LP L(shadow_stack_bound_recorded): /* If we unwind the stack, we can't undo stack unwinding. Just diff --git a/sysdeps/x86/cpu-features.c b/sysdeps/x86/cpu-features.c index 0bf923d48b..f180f0d9a4 100644 --- a/sysdeps/x86/cpu-features.c +++ b/sysdeps/x86/cpu-features.c @@ -1121,8 +1121,9 @@ no_cpuid: # ifndef SHARED /* Check if IBT and SHSTK are enabled by kernel. */ - if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT) - || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK)) + if ((cet_status + & (GNU_PROPERTY_X86_FEATURE_1_IBT + | GNU_PROPERTY_X86_FEATURE_1_SHSTK))) { /* Disable IBT and/or SHSTK if they are enabled by kernel, but disabled by environment variable: @@ -1131,9 +1132,11 @@ no_cpuid: */ unsigned int cet_feature = 0; if (!CPU_FEATURE_USABLE (IBT)) - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT; + cet_feature |= (cet_status + & GNU_PROPERTY_X86_FEATURE_1_IBT); if (!CPU_FEATURE_USABLE (SHSTK)) - cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; + cet_feature |= (cet_status + & GNU_PROPERTY_X86_FEATURE_1_SHSTK); if (cet_feature) { @@ -1148,7 +1151,9 @@ no_cpuid: lock CET if IBT or SHSTK is enabled permissively. */ if (GL(dl_x86_feature_control).ibt != cet_permissive && GL(dl_x86_feature_control).shstk != cet_permissive) - dl_cet_lock_cet (); + dl_cet_lock_cet (GL(dl_x86_feature_1) + & (GNU_PROPERTY_X86_FEATURE_1_IBT + | GNU_PROPERTY_X86_FEATURE_1_SHSTK)); } # endif } diff --git a/sysdeps/x86/dl-cet.c b/sysdeps/x86/dl-cet.c index e486e549be..66a78244d4 100644 --- a/sysdeps/x86/dl-cet.c +++ b/sysdeps/x86/dl-cet.c @@ -202,7 +202,7 @@ dl_cet_check_startup (struct link_map *m, struct dl_cet_info *info) feature_1_lock |= GNU_PROPERTY_X86_FEATURE_1_SHSTK; if (feature_1_lock != 0 - && dl_cet_lock_cet () != 0) + && dl_cet_lock_cet (feature_1_lock) != 0) _dl_fatal_printf ("%s: can't lock CET\n", info->program); } diff --git a/sysdeps/x86_64/nptl/tls.h b/sysdeps/x86_64/nptl/tls.h index 1403f939f7..4bcc2552a1 100644 --- a/sysdeps/x86_64/nptl/tls.h +++ b/sysdeps/x86_64/nptl/tls.h @@ -60,7 +60,7 @@ typedef struct void *__private_tm[4]; /* GCC split stack support. */ void *__private_ss; - /* The lowest address of shadow stack, */ + /* The marker for the current shadow stack. */ unsigned long long int ssp_base; /* Must be kept even if it is no longer used by glibc since programs, like AddressSanitizer, depend on the size of tcbhead_t. */