@@ -121,6 +121,7 @@ static inline u64 sc_retry(sc_func_t func, u64 fn,
int tdx_cpu_enable(void);
int tdx_enable(void);
const char *tdx_dump_mce_info(struct mce *m);
+void tdx_reset_memory(void);
struct tdx_metadata_field_mapping {
u64 field_id;
@@ -148,6 +149,7 @@ static inline void tdx_init(void) { }
static inline int tdx_cpu_enable(void) { return -ENODEV; }
static inline int tdx_enable(void) { return -ENODEV; }
static inline const char *tdx_dump_mce_info(struct mce *m) { return NULL; }
+static inline void tdx_reset_memory(void) { }
#endif /* CONFIG_INTEL_TDX_HOST */
#endif /* !__ASSEMBLY__ */
@@ -28,6 +28,7 @@
#include <asm/setup.h>
#include <asm/set_memory.h>
#include <asm/cpu.h>
+#include <asm/tdx.h>
#ifdef CONFIG_ACPI
/*
@@ -288,6 +289,14 @@ void machine_kexec_cleanup(struct kimage *image)
free_transition_pgtable(image);
}
+static void kexec_save_processor_start(struct kimage *image)
+{
+#ifdef CONFIG_KEXEC_JUMP
+ if (image->preserve_context)
+ save_processor_state();
+#endif
+}
+
/*
* Do not allocate memory (or fail in any way) in machine_kexec().
* We are past the point of no return, committed to rebooting now.
@@ -298,10 +307,20 @@ void machine_kexec(struct kimage *image)
void *control_page;
int save_ftrace_enabled;
-#ifdef CONFIG_KEXEC_JUMP
- if (image->preserve_context)
- save_processor_state();
-#endif
+ kexec_save_processor_start(image);
+
+ /*
+ * Convert TDX private memory back to normal (when needed) to
+ * avoid the second kernel potentially seeing unexpected machine
+ * check.
+ *
+ * However skip this when preserve_context is on. By reaching
+ * here, TDX (if ever got enabled by the kernel) has survived
+ * from the suspend when preserve_context is on, and it can
+ * continue to work after jumping back from the second kernel.
+ */
+ if (!image->preserve_context)
+ tdx_reset_memory();
save_ftrace_enabled = __ftrace_enabled_save();
@@ -61,6 +61,8 @@ static DEFINE_MUTEX(tdx_module_lock);
/* All TDX-usable memory regions. Protected by mem_hotplug_lock. */
static LIST_HEAD(tdx_memlist);
+static bool tdx_may_have_private_memory __read_mostly;
+
typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args);
static inline void seamcall_err(u64 fn, u64 err, struct tdx_module_args *args)
@@ -1172,6 +1174,18 @@ static int init_tdmrs(struct tdmr_info_list *tdmr_list)
return 0;
}
+static void mark_may_have_private_memory(bool may)
+{
+ tdx_may_have_private_memory = may;
+
+ /*
+ * Ensure update to tdx_may_have_private_memory is visible to all
+ * cpus. This ensures when any remote cpu reads it as true, the
+ * 'tdx_tdmr_list' must be stable for reading PAMTs.
+ */
+ smp_wmb();
+}
+
static int init_tdx_module(void)
{
struct tdx_tdmr_sysinfo tdmr_sysinfo;
@@ -1242,6 +1256,12 @@ static int init_tdx_module(void)
if (ret)
goto err_reset_pamts;
+ /*
+ * Starting from this point the system is possible to have
+ * TDX private memory.
+ */
+ mark_may_have_private_memory(true);
+
/* Initialize TDMRs to complete the TDX module initialization */
ret = init_tdmrs(&tdx_tdmr_list);
if (ret)
@@ -1280,6 +1300,7 @@ static int init_tdx_module(void)
* as suggested by the TDX spec.
*/
tdmrs_reset_pamt_all(&tdx_tdmr_list);
+ mark_may_have_private_memory(false);
err_free_pamts:
tdmrs_free_pamt_all(&tdx_tdmr_list);
err_free_tdmrs:
@@ -1597,3 +1618,61 @@ void __init tdx_init(void)
check_tdx_erratum();
}
+
+void tdx_reset_memory(void)
+{
+ if (!boot_cpu_has(X86_FEATURE_TDX_HOST_PLATFORM))
+ return;
+
+ /*
+ * Converting TDX private pages back to normal must be done
+ * when there's no TDX activity anymore on all remote cpus.
+ * Verify this is only called when all remote cpus have
+ * been stopped.
+ */
+ WARN_ON_ONCE(num_online_cpus() != 1);
+
+ /*
+ * Kernel read/write to TDX private memory doesn't cause
+ * machine check on hardware w/o this erratum.
+ */
+ if (!boot_cpu_has_bug(X86_BUG_TDX_PW_MCE))
+ return;
+
+ /*
+ * Nothing to convert if it's not possible to have any TDX
+ * private pages.
+ */
+ if (!tdx_may_have_private_memory)
+ return;
+
+ /*
+ * Ensure the 'tdx_tdmr_list' is stable for reading PAMTs
+ * when tdx_may_have_private_memory reads true, paired with
+ * the smp_wmb() in mark_may_have_private_memory().
+ */
+ smp_rmb();
+
+ /*
+ * All remote cpus have been stopped, and their caches have
+ * been flushed in stop_this_cpu(). Now flush cache for the
+ * last running cpu _before_ converting TDX private pages.
+ */
+ native_wbinvd();
+
+ /*
+ * It's ideal to cover all types of TDX private pages here, but
+ * currently there's no unified way to tell whether a given page
+ * is TDX private page or not.
+ *
+ * Just convert PAMT pages now, as currently TDX private pages
+ * can only be PAMT pages.
+ *
+ * TODO:
+ *
+ * This leaves all other types of TDX private pages undealt
+ * with. They must be handled in _some_ way when they become
+ * possible to exist.
+ */
+ tdmrs_reset_pamt_all(&tdx_tdmr_list);
+}