diff --git a/asm/misc.S b/asm/misc.S
index 647f60b2..55a9a99c 100644
--- a/asm/misc.S
+++ b/asm/misc.S
@@ -255,3 +255,40 @@ enter_p9_pm_state:
 	mtspr	SPR_PSSCR,%r3
 	PPC_INST_STOP
 	b	.
+
+/* Exit UV mode and disable Protected Execution Facility
+ * For each core, this should be run on all secondary threads first to bring
+ * them out of UV mode.  Then, it is called by the primary thread to disable
+ * PEF and bring it out of UV mode.  All threads will then be running in HV
+ * mode.  The only way to reenable UV mode is with a reboot.
+ * r3 = 1 if primary thread
+ *      0 if secondary thread
+ */
+.global exit_uv_mode
+exit_uv_mode:
+	mfmsr	%r4
+	LOAD_IMM64(%r5, ~MSR_S)
+	and	%r4,%r4,%r5
+	mtspr	SPR_USRR1,%r4
+
+	mfspr   %r4,SPR_HSRR1
+	and     %r4,%r4,%r5
+	mtspr   SPR_HSRR1,%r3
+
+	mfspr   %r4,SPR_SRR1
+	and     %r4,%r4,%r5
+	mtspr   SPR_SRR1,%r4
+
+	cmpdi	%r3,1
+	bne	1f
+	mfspr   %r4, SPR_SMFCTRL
+	LOAD_IMM64(%r5, ~PPC_BIT(0))
+	and     %r4,%r4,%r5
+	mtspr   SPR_SMFCTRL,%r4
+1:
+	isync
+
+	mflr	%r4
+	mtspr	SPR_USRR0,%r4
+
+	urfid
diff --git a/core/cpu.c b/core/cpu.c
index d5b7d623..1adf16cc 100644
--- a/core/cpu.c
+++ b/core/cpu.c
@@ -1644,3 +1644,62 @@ static int64_t opal_nmmu_set_ptcr(uint64_t chip_id, uint64_t ptcr)
 	return rc;
 }
 opal_call(OPAL_NMMU_SET_PTCR, opal_nmmu_set_ptcr, 2);
+
+static void _exit_uv_mode(void *data __unused)
+{
+	prlog(PR_DEBUG, "Exit uv mode on cpu pir 0x%04x\n", this_cpu()->pir);
+	/* HW has smfctrl shared between threads but on Mambo it is per-thread */
+	if (chip_quirk(QUIRK_MAMBO_CALLOUTS))
+		exit_uv_mode(1);
+	else
+		exit_uv_mode(cpu_is_thread0(this_cpu()));
+}
+
+void cpu_disable_pef(void)
+{
+	struct cpu_thread *cpu;
+	struct cpu_job **jobs;
+
+	if (!(mfmsr() & MSR_S)) {
+		prlog(PR_DEBUG, "UV mode off on cpu pir 0x%04x\n", this_cpu()->pir);
+		return;
+	}
+
+	jobs = zalloc(sizeof(struct cpu_job *) * (cpu_max_pir + 1));
+	assert(jobs);
+
+	/* Exit uv mode on all secondary threads before touching
+	 * smfctrl on thread 0 */
+	for_each_available_cpu(cpu) {
+		if (cpu == this_cpu())
+			continue;
+
+		if (!cpu_is_thread0(cpu))
+			jobs[cpu->pir] = cpu_queue_job(cpu, "exit_uv_mode",
+					_exit_uv_mode, NULL);
+	}
+
+	for_each_available_cpu(cpu)
+		if (jobs[cpu->pir]) {
+			cpu_wait_job(jobs[cpu->pir], true);
+			jobs[cpu->pir] = NULL;
+		}
+
+	/* Exit uv mode and disable smfctrl on primary threads */
+	for_each_available_cpu(cpu) {
+		if (cpu == this_cpu())
+			continue;
+
+		if (cpu_is_thread0(cpu))
+			jobs[cpu->pir] = cpu_queue_job(cpu, "exit_uv_mode",
+					_exit_uv_mode, NULL);
+	}
+
+	for_each_available_cpu(cpu)
+		if (jobs[cpu->pir])
+			cpu_wait_job(jobs[cpu->pir], true);
+
+	free(jobs);
+
+	_exit_uv_mode(NULL);
+}
diff --git a/core/init.c b/core/init.c
index 339462e5..0d993abb 100644
--- a/core/init.c
+++ b/core/init.c
@@ -1354,6 +1354,9 @@ void __noreturn __nomcount main_cpu_entry(const void *fdt)
 	/* Add the list of interrupts going to OPAL */
 	add_opal_interrupts();
 
+	/* Disable protected execution facility in BML */
+	cpu_disable_pef();
+
 	/* Now release parts of memory nodes we haven't used ourselves... */
 	mem_region_release_unused();
 
diff --git a/include/cpu.h b/include/cpu.h
index 686310d7..cab63360 100644
--- a/include/cpu.h
+++ b/include/cpu.h
@@ -309,4 +309,7 @@ int dctl_set_special_wakeup(struct cpu_thread *t);
 int dctl_clear_special_wakeup(struct cpu_thread *t);
 int dctl_core_is_gated(struct cpu_thread *t);
 
+extern void exit_uv_mode(int);
+void cpu_disable_pef(void);
+
 #endif /* __CPU_H */
diff --git a/include/processor.h b/include/processor.h
index a0c2864a..1fdcc02b 100644
--- a/include/processor.h
+++ b/include/processor.h
@@ -11,6 +11,7 @@
 #define MSR_HV		PPC_BIT(3)	/* Hypervisor mode */
 #define MSR_VEC		PPC_BIT(38)	/* VMX enable */
 #define MSR_VSX		PPC_BIT(40)	/* VSX enable */
+#define MSR_S		PPC_BIT(41)	/* Secure mode */
 #define MSR_EE		PPC_BIT(48)	/* External Int. Enable */
 #define MSR_PR		PPC_BIT(49)       	/* Problem state */
 #define MSR_FP		PPC_BIT(50)	/* Floating Point Enable */
@@ -65,6 +66,9 @@
 #define SPR_HMEER	0x151	/* HMER interrupt enable mask */
 #define SPR_PCR		0x152
 #define SPR_AMOR	0x15d
+#define SPR_USRR0	0x1fa   /* RW: Ultravisor Save/Restore Register 0 */
+#define SPR_USRR1	0x1fb   /* RW: Ultravisor Save/Restore Register 1 */
+#define SPR_SMFCTRL	0x1ff   /* RW: Secure Memory Facility Control */
 #define SPR_PSSCR	0x357   /* RW: Stop status and control (ISA 3) */
 #define SPR_TSCR	0x399
 #define SPR_HID0	0x3f0
