@@ -11,7 +11,7 @@ config UML
select ARCH_HAS_KCOV
select ARCH_HAS_STRNCPY_FROM_USER
select ARCH_HAS_STRNLEN_USER
- select ARCH_NO_PREEMPT
+ select ARCH_NO_PREEMPT_DYNAMIC
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_KASAN if X86_64
select HAVE_ARCH_KASAN_VMALLOC if HAVE_ARCH_KASAN
@@ -4,12 +4,15 @@
/* Copyright (c) 2020 Cambridge Greys Ltd
* Copyright (c) 2020 Red Hat Inc.
- * A set of "dummy" defines to allow the direct inclusion
- * of x86 optimized copy, xor, etc routines into the
- * UML code tree. */
+ */
+#if defined(CONFIG_PREEMPT) || defined(CONFIG_PREEMPT_VOLUNTARY)
+extern void kernel_fpu_begin(void);
+extern void kernel_fpu_end(void);
+#else
#define kernel_fpu_begin() (void)0
#define kernel_fpu_end() (void)0
+#endif
static inline bool irq_fpu_usable(void)
{
@@ -23,8 +23,8 @@ struct thread_info {
int preempt_count; /* 0 => preemptable,
<0 => BUG */
struct thread_info *real_thread; /* Points to non-IRQ stack */
- unsigned long aux_fp_regs[FP_SIZE]; /* auxiliary fp_regs to save/restore
- them out-of-band */
+ /* auxiliary fp_regs to save/restore them out-of-band */
+ unsigned long aux_fp_regs[FP_SIZE] __aligned(64);
};
#define INIT_THREAD_INFO(tsk) \
@@ -26,9 +26,13 @@ obj-$(CONFIG_OF) += dtb.o
obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
obj-$(CONFIG_STACKTRACE) += stacktrace.o
obj-$(CONFIG_GENERIC_PCI_IOMAP) += ioport.o
+obj-$(CONFIG_PREEMPT) += fpu.o
+obj-$(CONFIG_PREEMPT_VOLUNTARY) += fpu.o
USER_OBJS := config.o
+CFLAGS_fpu.o += -mxsave -mxsaveopt
+
include $(srctree)/arch/um/scripts/Makefile.rules
targets := config.c config.tmp capflags.c
new file mode 100644
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2023 Cambridge Greys Ltd
+ * Copyright (C) 2023 Red Hat Inc
+ */
+
+#include <linux/cpu.h>
+#include <linux/init.h>
+#include <asm/fpu/api.h>
+#include <asm/cpufeature.h>
+
+/*
+ * The critical section between kernel_fpu_begin() and kernel_fpu_end()
+ * is non-reentrant. It is the caller's responsibility to avoid reentrance.
+ */
+
+static DEFINE_PER_CPU(bool, in_kernel_fpu);
+
+/* UML and driver code it pulls out of the x86 tree knows about 387 features
+ * up to and including AVX512. TILE, etc are not yet supported.
+ */
+
+#define KNOWN_387_FEATURES 0xFF
+
+void kernel_fpu_begin(void)
+{
+ preempt_disable();
+
+ WARN_ON(this_cpu_read(in_kernel_fpu));
+
+ this_cpu_write(in_kernel_fpu, true);
+
+#ifdef CONFIG_64BIT
+ if (likely(cpu_has(&boot_cpu_data, X86_FEATURE_XSAVEOPT)))
+ __builtin_ia32_xsaveopt64(¤t_thread_info()->aux_fp_regs, KNOWN_387_FEATURES);
+ else if (likely(cpu_has(&boot_cpu_data, X86_FEATURE_XSAVE)))
+ __builtin_ia32_xsave64(¤t_thread_info()->aux_fp_regs, KNOWN_387_FEATURES);
+ else
+ __builtin_ia32_fxsave64(¤t_thread_info()->aux_fp_regs);
+#else
+ if (likely(cpu_has(&boot_cpu_data, X86_FEATURE_XSAVEOPT)))
+ __builtin_ia32_xsaveopt(¤t->aux_fp_regs, KNOWN_387_FEATURES);
+ else if (likely(cpu_has(&boot_cpu_data, X86_FEATURE_XSAVE)))
+ __builtin_ia32_xsave(¤t->aux_fp_regs, KNOWN_387_FEATURES);
+ else
+ __builtin_ia32_fxsave(¤t->aux_fp_regs);
+#endif
+}
+EXPORT_SYMBOL_GPL(kernel_fpu_begin);
+
+void kernel_fpu_end(void)
+{
+ WARN_ON(!this_cpu_read(in_kernel_fpu));
+
+#ifdef CONFIG_64BIT
+ if (likely(cpu_has(&boot_cpu_data, X86_FEATURE_XSAVE)))
+ __builtin_ia32_xrstor64(¤t_thread_info()->aux_fp_regs, KNOWN_387_FEATURES);
+ else
+ __builtin_ia32_fxrstor64(¤t_thread_info()->aux_fp_regs);
+#else
+ if (likely(cpu_has(&boot_cpu_data, X86_FEATURE_XSAVE)))
+ __builtin_ia32_xrstor(¤t_tread_info()->aux_fp_regs, KNOWN_387_FEATURES);
+ else
+ __builtin_ia32_fxrstor(¤t_thread_info()->aux_fp_regs);
+#endif
+ this_cpu_write(in_kernel_fpu, false);
+
+ preempt_enable();
+}
+EXPORT_SYMBOL_GPL(kernel_fpu_end);
+
@@ -195,7 +195,9 @@ static void _sigio_handler(struct uml_pt_regs *regs,
void sigio_handler(int sig, struct siginfo *unused_si, struct uml_pt_regs *regs)
{
+ preempt_disable();
_sigio_handler(regs, irqs_suspended);
+ preempt_enable();
}
static struct irq_entry *get_irq_entry_by_fd(int fd)
@@ -322,6 +322,8 @@ static void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
unsigned long addr = start_addr, next;
int ret = 0, userspace = 1;
+ preempt_disable();
+
hvc = INIT_HVC(mm, force, userspace);
pgd = pgd_offset(mm, addr);
do {
@@ -346,6 +348,7 @@ static void fix_range_common(struct mm_struct *mm, unsigned long start_addr,
"process: %d\n", task_tgid_vnr(current));
mm_idp->kill = 1;
}
+ preempt_enable();
}
static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
@@ -362,6 +365,9 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
mm = &init_mm;
hvc = INIT_HVC(mm, force, userspace);
+
+ preempt_disable();
+
for (addr = start; addr < end;) {
pgd = pgd_offset(mm, addr);
if (!pgd_present(*pgd)) {
@@ -449,6 +455,9 @@ static int flush_tlb_kernel_range_common(unsigned long start, unsigned long end)
if (err < 0)
panic("flush_tlb_kernel failed, errno = %d\n", err);
+
+ preempt_enable();
+
return updated;
}
@@ -466,6 +475,8 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
address &= PAGE_MASK;
+ preempt_disable();
+
pgd = pgd_offset(mm, address);
if (!pgd_present(*pgd))
goto kill;
@@ -520,6 +531,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long address)
*pte = pte_mkuptodate(*pte);
+ preempt_enable();
return;
kill: