@@ -12,6 +12,9 @@
extern struct static_key_false disable_kuap_key;
extern struct static_key_false disable_kuep_key;
+extern s32 patch__kuep_interrupt_entry, patch__kuep_interrupt_exit;
+extern s32 patch__kuep_syscall_entry, patch__kuep_syscall_exit;
+
static __always_inline bool kuap_is_disabled(void)
{
return !IS_ENABLED(CONFIG_PPC_KUAP) || static_branch_unlikely(&disable_kuap_key);
@@ -196,6 +196,7 @@ struct hash_pte {
typedef struct {
unsigned long id;
+ unsigned long sr0;
void __user *vdso;
} mm_context_t;
@@ -157,6 +157,7 @@ struct thread_struct {
#ifdef CONFIG_PPC_BOOK3S_32
unsigned long r0, r3, r4, r5, r6, r8, r9, r11;
unsigned long lr, ctr;
+ unsigned long sr0;
#endif
#endif /* CONFIG_PPC32 */
/* Debug Registers */
@@ -33,6 +33,7 @@
#include <asm/kup.h>
#include <asm/bug.h>
#include <asm/interrupt.h>
+#include <asm/code-patching-asm.h>
#include "head_32.h"
@@ -55,11 +56,10 @@ prepare_transfer_to_handler:
#ifdef CONFIG_PPC_KUEP
beq 1f
- mfsr r4,0
- rlwinm r4,r4,0,8,3
- oris r4,r4,SR_NX@h
+0: blr /* lwz r4, current->thread.sr0(r2) */
update_user_segments_by_6 r4, r5, r6, r7, r8, r9
blr
+ patch_site 0b, patch__kuep_interrupt_entry
1:
#endif
/* if from kernel, check interrupted DOZE/NAP mode */
@@ -104,10 +104,10 @@ transfer_to_syscall:
SAVE_2GPRS(7, r1)
addi r2,r10,-THREAD
#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
- mfsr r9,0
- rlwinm r9,r9,0,8,3
- oris r9,r9,SR_NX@h
+0: b 1f /* lwz r9, &thread.sr0(r10) */
update_user_segments_by_4 r9, r10, r11, r12
+ patch_site 0b, patch__kuep_syscall_entry
+1:
#endif
SAVE_NVGPRS(r1)
@@ -127,9 +127,11 @@ ret_from_syscall:
bne- 2f
#endif /* CONFIG_PPC_47x */
#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
- mfsr r7,0
- rlwinm r7,r7,0,8,2
+0: b 1f /* lwz r7, current->thread.sr0(r2) */
+ rlwinm r7,r7,0,~SR_NX
update_user_segments_by_6 r7, r8, r9, r10, r11, r12
+ patch_site 0b, patch__kuep_syscall_exit
+1:
#endif
lwz r4,_LINK(r1)
lwz r5,_CCR(r1)
@@ -295,9 +297,11 @@ interrupt_return:
bl interrupt_exit_user_prepare
cmpwi r3,0
#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
- mfsr r7,0
- rlwinm r7,r7,0,8,2
+0: b 1f /* lwz r7, current->thread.sr0(r2) */
+ rlwinm r7,r7,0,~SR_NX
update_user_segments_by_6 r7, r8, r9, r10, r11, r12
+ patch_site 0b, patch__kuep_interrupt_exit
+1:
#endif
bne- .Lrestore_nvgprs
@@ -20,8 +20,11 @@ EXPORT_SYMBOL(kuap_unlock_all_ool);
void setup_kuap(bool disabled)
{
- if (!disabled)
+ if (!disabled) {
kuap_lock_all_ool();
+ init_mm.context.sr0 |= SR_KS;
+ current->thread.sr0 |= SR_KS;
+ }
if (smp_processor_id() != boot_cpuid)
return;
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-or-later
+#include <asm/code-patching.h>
#include <asm/kup.h>
#include <asm/smp.h>
@@ -7,19 +8,32 @@ struct static_key_false disable_kuep_key;
void setup_kuep(bool disabled)
{
- if (disabled) {
- pr_info("KUEP cannot be disabled for the time being\n");
- disabled = false;
- }
+ u32 insn;
- if (!disabled)
- update_user_segments(mfsr(0) | SR_NX);
+ if (!disabled) {
+ init_mm.context.sr0 |= SR_NX;
+ current->thread.sr0 |= SR_NX;
+ update_user_segments(init_mm.context.sr0);
+ }
if (smp_processor_id() != boot_cpuid)
return;
if (disabled)
static_branch_enable(&disable_kuep_key);
- else
- pr_info("Activating Kernel Userspace Execution Prevention\n");
+
+ if (disabled)
+ return;
+
+ insn = PPC_RAW_LWZ(_R4, _R2, offsetof(struct task_struct, thread.sr0));
+ patch_instruction_site(&patch__kuep_interrupt_entry, ppc_inst(insn));
+
+ insn = PPC_RAW_LWZ(_R9, _R10, offsetof(struct thread_struct, sr0));
+ patch_instruction_site(&patch__kuep_syscall_entry, ppc_inst(insn));
+
+ insn = PPC_RAW_LWZ(_R7, _R2, offsetof(struct task_struct, thread.sr0));
+ patch_instruction_site(&patch__kuep_syscall_exit, ppc_inst(insn));
+ patch_instruction_site(&patch__kuep_interrupt_exit, ppc_inst(insn));
+
+ pr_info("Activating Kernel Userspace Execution Prevention\n");
}
@@ -69,6 +69,12 @@ EXPORT_SYMBOL_GPL(__init_new_context);
int init_new_context(struct task_struct *t, struct mm_struct *mm)
{
mm->context.id = __init_new_context();
+ mm->context.sr0 = CTX_TO_VSID(mm->context.id, 0);
+
+ if (!kuep_is_disabled())
+ mm->context.sr0 |= SR_NX;
+ if (!kuap_is_disabled())
+ mm->context.sr0 |= SR_KS;
return 0;
}
@@ -108,20 +114,13 @@ void __init mmu_context_init(void)
void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk)
{
long id = next->context.id;
- unsigned long val;
if (id < 0)
panic("mm_struct %p has no context ID", next);
isync();
- val = CTX_TO_VSID(id, 0);
- if (!kuep_is_disabled())
- val |= SR_NX;
- if (!kuap_is_disabled())
- val |= SR_KS;
-
- update_user_segments(val);
+ update_user_segments(next->context.sr0);
if (IS_ENABLED(CONFIG_BDI_SWITCH))
abatron_pteptrs[1] = next->pgd;
@@ -18,6 +18,9 @@ static inline void switch_mm_pgdir(struct task_struct *tsk,
{
/* 32-bit keeps track of the current PGDIR in the thread struct */
tsk->thread.pgdir = mm->pgd;
+#ifdef CONFIG_PPC_BOOK3S_32
+ tsk->thread.sr0 = mm->context.sr0;
+#endif
}
#elif defined(CONFIG_PPC_BOOK3E_64)
static inline void switch_mm_pgdir(struct task_struct *tsk,
Calling 'mfsr' to get the content of segment registers is heavy, in addition it requires clearing of the 'reserved' bits. In order to avoid this operation, save it in mm context and in thread struct. The saved sr0 is the one used by kernel, this means that on interrupt/syscall entry it can be used as is. In interrupt/syscall exit, the only thing to do is to clear SR_NX. This improves null_syscall selftest by 12 cycles, ie 4%. Capability to deactive KUEP at boot time is re-enabled by this patch. Signed-off-by: Christophe Leroy <christophe.leroy@csgroup.eu> --- arch/powerpc/include/asm/book3s/32/kup.h | 3 ++ arch/powerpc/include/asm/book3s/32/mmu-hash.h | 1 + arch/powerpc/include/asm/processor.h | 1 + arch/powerpc/kernel/entry_32.S | 24 ++++++++------- arch/powerpc/mm/book3s32/kuap.c | 5 +++- arch/powerpc/mm/book3s32/kuep.c | 30 ++++++++++++++----- arch/powerpc/mm/book3s32/mmu_context.c | 15 +++++----- arch/powerpc/mm/mmu_context.c | 3 ++ 8 files changed, 55 insertions(+), 27 deletions(-)