@@ -14,8 +14,16 @@
#ifdef CONFIG_EFI
extern void efi_init(void);
+
+bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg);
#else
#define efi_init()
+
+static inline
+bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg)
+{
+ return false;
+}
#endif
int efi_create_mapping(struct mm_struct *mm, efi_memory_desc_t *md);
@@ -7,7 +7,7 @@
#include <asm/assembler.h>
SYM_FUNC_START(__efi_rt_asm_wrapper)
- stp x29, x30, [sp, #-32]!
+ stp x29, x30, [sp, #-112]!
mov x29, sp
/*
@@ -17,11 +17,21 @@ SYM_FUNC_START(__efi_rt_asm_wrapper)
*/
stp x1, x18, [sp, #16]
+ /*
+ * Preserve all callee saved registers and preserve the stack pointer
+ * value at the base of the EFI runtime stack so we can recover from
+ * synchronous exceptions occurring while executing the firmware
+ * routines.
+ */
+ stp x19, x20, [sp, #32]
+ stp x21, x22, [sp, #48]
+ stp x23, x24, [sp, #64]
+ stp x25, x26, [sp, #80]
+ stp x27, x28, [sp, #96]
+
ldr_l x16, efi_rt_stack_top
mov sp, x16
-#ifdef CONFIG_SHADOW_CALL_STACK
- str x18, [sp, #-16]!
-#endif
+ stp x18, x29, [sp, #-16]!
/*
* We are lucky enough that no EFI runtime services take more than
@@ -39,7 +49,7 @@ SYM_FUNC_START(__efi_rt_asm_wrapper)
mov sp, x29
ldp x1, x2, [sp, #16]
cmp x2, x18
- ldp x29, x30, [sp], #32
+ ldp x29, x30, [sp], #112
b.ne 0f
ret
0:
@@ -57,3 +67,15 @@ SYM_FUNC_START(__efi_rt_asm_wrapper)
b efi_handle_corrupted_x18 // tail call
SYM_FUNC_END(__efi_rt_asm_wrapper)
+
+SYM_CODE_START(__efi_rt_asm_recover)
+ mov sp, x30
+
+ ldp x19, x20, [sp, #32]
+ ldp x21, x22, [sp, #48]
+ ldp x23, x24, [sp, #64]
+ ldp x25, x26, [sp, #80]
+ ldp x27, x28, [sp, #96]
+ ldp x29, x30, [sp], #112
+ ret
+SYM_CODE_END(__efi_rt_asm_recover)
@@ -149,6 +149,28 @@ DEFINE_SPINLOCK(efi_rt_lock);
asmlinkage u64 *efi_rt_stack_top __ro_after_init;
+asmlinkage efi_status_t __efi_rt_asm_recover(void);
+
+bool efi_runtime_fixup_exception(struct pt_regs *regs, const char *msg)
+{
+ /* Check whether the exception occurred while running the firmware */
+ if (current_work() != &efi_rts_work.work || regs->pc >= TASK_SIZE_64)
+ return false;
+
+ pr_err(FW_BUG "Unable to handle %s in EFI runtime service\n", msg);
+ add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
+ clear_bit(EFI_RUNTIME_SERVICES, &efi.flags);
+
+ regs->regs[0] = EFI_ABORTED;
+ regs->regs[30] = efi_rt_stack_top[-1];
+ regs->pc = (u64)__efi_rt_asm_recover;
+
+ if (IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
+ regs->regs[18] = efi_rt_stack_top[-2];
+
+ return true;
+}
+
/* EFI requires 8 KiB of stack space for runtime services */
static_assert(THREAD_SIZE >= SZ_8K);
@@ -30,6 +30,7 @@
#include <asm/bug.h>
#include <asm/cmpxchg.h>
#include <asm/cpufeature.h>
+#include <asm/efi.h>
#include <asm/exception.h>
#include <asm/daifflags.h>
#include <asm/debug-monitors.h>
@@ -397,6 +398,9 @@ static void __do_kernel_fault(unsigned long addr, unsigned long esr,
msg = "paging request";
}
+ if (efi_runtime_fixup_exception(regs, msg))
+ return;
+
die_kernel_fault(msg, addr, esr, regs);
}
@@ -84,6 +84,7 @@ struct efi_runtime_work efi_rts_work;
else \
pr_err("Failed to queue work to efi_rts_wq.\n"); \
\
+ WARN_ON_ONCE(efi_rts_work.status == EFI_ABORTED); \
exit: \
efi_rts_work.efi_rts_id = EFI_NONE; \
efi_rts_work.status; \