diff mbox series

[v2,4/4] Add the NVMM acceleration enlightenments

Message ID 20200128140945.929-5-n54@gmx.com
State New
Headers show
Series [v2,1/4] Add the NVMM vcpu API | expand

Commit Message

Kamil Rytarowski Jan. 28, 2020, 2:09 p.m. UTC
From: Maxime Villard <max@m00nbsd.net>

Implements the NVMM accelerator cpu enlightenments to actually use the nvmm-all
accelerator on NetBSD platforms.

Signed-off-by: Maxime Villard <max@m00nbsd.net>
Signed-off-by: Kamil Rytarowski <n54@gmx.com>
Reviewed-by: Sergio Lopez <slp@redhat.com>
---
 cpus.c                    | 58 +++++++++++++++++++++++++++++++++++++++
 include/sysemu/hw_accel.h | 14 ++++++++++
 target/i386/helper.c      |  2 +-
 3 files changed, 73 insertions(+), 1 deletion(-)

--
2.24.1

Comments

Philippe Mathieu-Daudé Feb. 3, 2020, 11:54 a.m. UTC | #1
On 1/28/20 3:09 PM, Kamil Rytarowski wrote:
> From: Maxime Villard <max@m00nbsd.net>
> 
> Implements the NVMM accelerator cpu enlightenments to actually use the nvmm-all
> accelerator on NetBSD platforms.
> 
> Signed-off-by: Maxime Villard <max@m00nbsd.net>
> Signed-off-by: Kamil Rytarowski <n54@gmx.com>
> Reviewed-by: Sergio Lopez <slp@redhat.com>
> ---
>   cpus.c                    | 58 +++++++++++++++++++++++++++++++++++++++
>   include/sysemu/hw_accel.h | 14 ++++++++++
>   target/i386/helper.c      |  2 +-
>   3 files changed, 73 insertions(+), 1 deletion(-)
> 
> diff --git a/cpus.c b/cpus.c
> index b472378b70..3c3f63588c 100644
> --- a/cpus.c
> +++ b/cpus.c
> @@ -42,6 +42,7 @@
>   #include "sysemu/hax.h"
>   #include "sysemu/hvf.h"
>   #include "sysemu/whpx.h"
> +#include "sysemu/nvmm.h"
>   #include "exec/exec-all.h"
> 
>   #include "qemu/thread.h"
> @@ -1666,6 +1667,48 @@ static void *qemu_whpx_cpu_thread_fn(void *arg)
>       return NULL;
>   }
> 
> +static void *qemu_nvmm_cpu_thread_fn(void *arg)
> +{
> +    CPUState *cpu = arg;
> +    int r;
> +
> +    assert(nvmm_enabled());
> +
> +    rcu_register_thread();
> +
> +    qemu_mutex_lock_iothread();
> +    qemu_thread_get_self(cpu->thread);
> +    cpu->thread_id = qemu_get_thread_id();
> +    current_cpu = cpu;
> +
> +    r = nvmm_init_vcpu(cpu);
> +    if (r < 0) {
> +        fprintf(stderr, "nvmm_init_vcpu failed: %s\n", strerror(-r));
> +        exit(1);
> +    }
> +
> +    /* signal CPU creation */
> +    cpu->created = true;
> +    qemu_cond_signal(&qemu_cpu_cond);
> +
> +    do {
> +        if (cpu_can_run(cpu)) {
> +            r = nvmm_vcpu_exec(cpu);
> +            if (r == EXCP_DEBUG) {
> +                cpu_handle_guest_debug(cpu);
> +            }
> +        }
> +        qemu_wait_io_event(cpu);
> +    } while (!cpu->unplug || cpu_can_run(cpu));
> +
> +    nvmm_destroy_vcpu(cpu);
> +    cpu->created = false;
> +    qemu_cond_signal(&qemu_cpu_cond);
> +    qemu_mutex_unlock_iothread();
> +    rcu_unregister_thread();
> +    return NULL;
> +}
> +
>   #ifdef _WIN32
>   static void CALLBACK dummy_apc_func(ULONG_PTR unused)
>   {
> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>   #endif
>   }
> 
> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
> +{
> +    char thread_name[VCPU_THREAD_NAME_SIZE];
> +
> +    cpu->thread = g_malloc0(sizeof(QemuThread));
> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));

Nitpick, we prefer g_new0().

Reviewed-by: Philippe Mathieu-Daudé <philmd@redhat.com>

> +    qemu_cond_init(cpu->halt_cond);
> +    snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/NVMM",
> +             cpu->cpu_index);
> +    qemu_thread_create(cpu->thread, thread_name, qemu_nvmm_cpu_thread_fn,
> +                       cpu, QEMU_THREAD_JOINABLE);
> +}
> +
>   static void qemu_dummy_start_vcpu(CPUState *cpu)
>   {
>       char thread_name[VCPU_THREAD_NAME_SIZE];
> @@ -2069,6 +2125,8 @@ void qemu_init_vcpu(CPUState *cpu)
>           qemu_tcg_init_vcpu(cpu);
>       } else if (whpx_enabled()) {
>           qemu_whpx_start_vcpu(cpu);
> +    } else if (nvmm_enabled()) {
> +        qemu_nvmm_start_vcpu(cpu);
>       } else {
>           qemu_dummy_start_vcpu(cpu);
>       }
> diff --git a/include/sysemu/hw_accel.h b/include/sysemu/hw_accel.h
> index 0ec2372477..dbfa7a02f9 100644
> --- a/include/sysemu/hw_accel.h
> +++ b/include/sysemu/hw_accel.h
> @@ -15,6 +15,7 @@
>   #include "sysemu/hax.h"
>   #include "sysemu/kvm.h"
>   #include "sysemu/whpx.h"
> +#include "sysemu/nvmm.h"
> 
>   static inline void cpu_synchronize_state(CPUState *cpu)
>   {
> @@ -27,6 +28,9 @@ static inline void cpu_synchronize_state(CPUState *cpu)
>       if (whpx_enabled()) {
>           whpx_cpu_synchronize_state(cpu);
>       }
> +    if (nvmm_enabled()) {
> +        nvmm_cpu_synchronize_state(cpu);
> +    }
>   }
> 
>   static inline void cpu_synchronize_post_reset(CPUState *cpu)
> @@ -40,6 +44,10 @@ static inline void cpu_synchronize_post_reset(CPUState *cpu)
>       if (whpx_enabled()) {
>           whpx_cpu_synchronize_post_reset(cpu);
>       }
> +    if (nvmm_enabled()) {
> +        nvmm_cpu_synchronize_post_reset(cpu);
> +    }
> +
>   }
> 
>   static inline void cpu_synchronize_post_init(CPUState *cpu)
> @@ -53,6 +61,9 @@ static inline void cpu_synchronize_post_init(CPUState *cpu)
>       if (whpx_enabled()) {
>           whpx_cpu_synchronize_post_init(cpu);
>       }
> +    if (nvmm_enabled()) {
> +        nvmm_cpu_synchronize_post_init(cpu);
> +    }
>   }
> 
>   static inline void cpu_synchronize_pre_loadvm(CPUState *cpu)
> @@ -66,6 +77,9 @@ static inline void cpu_synchronize_pre_loadvm(CPUState *cpu)
>       if (whpx_enabled()) {
>           whpx_cpu_synchronize_pre_loadvm(cpu);
>       }
> +    if (nvmm_enabled()) {
> +        nvmm_cpu_synchronize_pre_loadvm(cpu);
> +    }
>   }
> 
>   #endif /* QEMU_HW_ACCEL_H */
> diff --git a/target/i386/helper.c b/target/i386/helper.c
> index c3a6e4fabe..2e79d61329 100644
> --- a/target/i386/helper.c
> +++ b/target/i386/helper.c
> @@ -981,7 +981,7 @@ void cpu_report_tpr_access(CPUX86State *env, TPRAccess access)
>       X86CPU *cpu = env_archcpu(env);
>       CPUState *cs = env_cpu(env);
> 
> -    if (kvm_enabled() || whpx_enabled()) {
> +    if (kvm_enabled() || whpx_enabled() || nvmm_enabled()) {
>           env->tpr_access_type = access;
> 
>           cpu_interrupt(cs, CPU_INTERRUPT_TPR);
> --
> 2.24.1
>
Kamil Rytarowski Feb. 6, 2020, 10:24 a.m. UTC | #2
On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>>   #endif
>>   }
>>
>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>> +{
>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>> +
>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>
> Nitpick, we prefer g_new0().

In this file other qemu_*_start_vcpu() use  g_malloc0().

I will leave this part unchanged and defer tor future style fixups if
someone is interested.
Philippe Mathieu-Daudé Feb. 6, 2020, 12:18 p.m. UTC | #3
On 2/6/20 11:24 AM, Kamil Rytarowski wrote:
> On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>>>    #endif
>>>    }
>>>
>>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>>> +{
>>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>>> +
>>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>>
>> Nitpick, we prefer g_new0().
> 
> In this file other qemu_*_start_vcpu() use  g_malloc0().
> 
> I will leave this part unchanged and defer tor future style fixups if
> someone is interested.

Fair enough.
Markus Armbruster Feb. 6, 2020, 1:06 p.m. UTC | #4
Kamil Rytarowski <n54@gmx.com> writes:

> On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>>>   #endif
>>>   }
>>>
>>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>>> +{
>>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>>> +
>>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>>
>> Nitpick, we prefer g_new0().
>
> In this file other qemu_*_start_vcpu() use  g_malloc0().
>
> I will leave this part unchanged and defer tor future style fixups if
> someone is interested.

Time to re-run Coccinelle with the semantic patch from commit
b45c03f585e.
Philippe Mathieu-Daudé Feb. 6, 2020, 1:09 p.m. UTC | #5
On Thu, Feb 6, 2020 at 2:06 PM Markus Armbruster <armbru@redhat.com> wrote:
> Kamil Rytarowski <n54@gmx.com> writes:
>
> > On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
> >>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
> >>>   #endif
> >>>   }
> >>>
> >>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
> >>> +{
> >>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
> >>> +
> >>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
> >>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
> >>
> >> Nitpick, we prefer g_new0().
> >
> > In this file other qemu_*_start_vcpu() use  g_malloc0().
> >
> > I will leave this part unchanged and defer tor future style fixups if
> > someone is interested.
>
> Time to re-run Coccinelle with the semantic patch from commit
> b45c03f585e.

I thought about it, but then noticed it would be clever to modify
checkpatch to refuse 'g_malloc0?(.*sizeof.*);'
Kamil Rytarowski Feb. 6, 2020, 1:31 p.m. UTC | #6
On 06.02.2020 14:09, Philippe Mathieu-Daudé wrote:
> On Thu, Feb 6, 2020 at 2:06 PM Markus Armbruster <armbru@redhat.com> wrote:
>> Kamil Rytarowski <n54@gmx.com> writes:
>>
>>> On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>>>>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>>>>>   #endif
>>>>>   }
>>>>>
>>>>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>>>>> +{
>>>>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>>>>> +
>>>>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>>>>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>>>>
>>>> Nitpick, we prefer g_new0().
>>>
>>> In this file other qemu_*_start_vcpu() use  g_malloc0().
>>>
>>> I will leave this part unchanged and defer tor future style fixups if
>>> someone is interested.
>>
>> Time to re-run Coccinelle with the semantic patch from commit
>> b45c03f585e.
>
> I thought about it, but then noticed it would be clever to modify
> checkpatch to refuse 'g_malloc0?(.*sizeof.*);'
>
>

As the patchset was reviewed, could we please merge it in the current
(v3) form (*) please?

Feel free to fixup the style after that as you like.

We plan to release NetBSD 9.0 in 1-2 weeks unless there will be a delay.

https://blog.netbsd.org/tnf/entry/second_final_release_candidate_for

(*) https://lists.gnu.org/archive/html/qemu-devel/2020-02/msg01405.html
Markus Armbruster Feb. 6, 2020, 2:13 p.m. UTC | #7
Kamil Rytarowski <n54@gmx.com> writes:

> On 06.02.2020 14:09, Philippe Mathieu-Daudé wrote:
>> On Thu, Feb 6, 2020 at 2:06 PM Markus Armbruster <armbru@redhat.com> wrote:
>>> Kamil Rytarowski <n54@gmx.com> writes:
>>>
>>>> On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>>>>>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>>>>>>   #endif
>>>>>>   }
>>>>>>
>>>>>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>>>>>> +{
>>>>>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>>>>>> +
>>>>>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>>>>>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>>>>>
>>>>> Nitpick, we prefer g_new0().
>>>>
>>>> In this file other qemu_*_start_vcpu() use  g_malloc0().
>>>>
>>>> I will leave this part unchanged and defer tor future style fixups if
>>>> someone is interested.
>>>
>>> Time to re-run Coccinelle with the semantic patch from commit
>>> b45c03f585e.
>>
>> I thought about it, but then noticed it would be clever to modify
>> checkpatch to refuse 'g_malloc0?(.*sizeof.*);'
>>
>>
>
> As the patchset was reviewed, could we please merge it in the current
> (v3) form (*) please?

No objection.  If I wanted you to clean this up before we accept your
work, I would've told you :)

[...]
Kamil Rytarowski Feb. 6, 2020, 3:38 p.m. UTC | #8
On 06.02.2020 15:13, Markus Armbruster wrote:
> Kamil Rytarowski <n54@gmx.com> writes:
>
>> On 06.02.2020 14:09, Philippe Mathieu-Daudé wrote:
>>> On Thu, Feb 6, 2020 at 2:06 PM Markus Armbruster <armbru@redhat.com> wrote:
>>>> Kamil Rytarowski <n54@gmx.com> writes:
>>>>
>>>>> On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>>>>>>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>>>>>>>   #endif
>>>>>>>   }
>>>>>>>
>>>>>>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>>>>>>> +{
>>>>>>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>>>>>>> +
>>>>>>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>>>>>>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>>>>>>
>>>>>> Nitpick, we prefer g_new0().
>>>>>
>>>>> In this file other qemu_*_start_vcpu() use  g_malloc0().
>>>>>
>>>>> I will leave this part unchanged and defer tor future style fixups if
>>>>> someone is interested.
>>>>
>>>> Time to re-run Coccinelle with the semantic patch from commit
>>>> b45c03f585e.
>>>
>>> I thought about it, but then noticed it would be clever to modify
>>> checkpatch to refuse 'g_malloc0?(.*sizeof.*);'
>>>
>>>
>>
>> As the patchset was reviewed, could we please merge it in the current
>> (v3) form (*) please?
>
> No objection.  If I wanted you to clean this up before we accept your
> work, I would've told you :)
>
> [...]
>
>

I see. I don't own myself a merge queue so I depend on yours.

Thank you in advance!
Philippe Mathieu-Daudé Feb. 6, 2020, 4:07 p.m. UTC | #9
On 2/6/20 4:38 PM, Kamil Rytarowski wrote:
> On 06.02.2020 15:13, Markus Armbruster wrote:
>> Kamil Rytarowski <n54@gmx.com> writes:
>>
>>> On 06.02.2020 14:09, Philippe Mathieu-Daudé wrote:
>>>> On Thu, Feb 6, 2020 at 2:06 PM Markus Armbruster <armbru@redhat.com> wrote:
>>>>> Kamil Rytarowski <n54@gmx.com> writes:
>>>>>
>>>>>> On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>>>>>>>> @@ -2029,6 +2072,19 @@ static void qemu_whpx_start_vcpu(CPUState *cpu)
>>>>>>>>    #endif
>>>>>>>>    }
>>>>>>>>
>>>>>>>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>>>>>>>> +{
>>>>>>>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>>>>>>>> +
>>>>>>>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>>>>>>>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>>>>>>>
>>>>>>> Nitpick, we prefer g_new0().
>>>>>>
>>>>>> In this file other qemu_*_start_vcpu() use  g_malloc0().
>>>>>>
>>>>>> I will leave this part unchanged and defer tor future style fixups if
>>>>>> someone is interested.
>>>>>
>>>>> Time to re-run Coccinelle with the semantic patch from commit
>>>>> b45c03f585e.
>>>>
>>>> I thought about it, but then noticed it would be clever to modify
>>>> checkpatch to refuse 'g_malloc0?(.*sizeof.*);'
>>>>
>>>>
>>>
>>> As the patchset was reviewed, could we please merge it in the current
>>> (v3) form (*) please?
>>
>> No objection.  If I wanted you to clean this up before we accept your
>> work, I would've told you :)
>>
>> [...]
>>
>>
> 
> I see. I don't own myself a merge queue so I depend on yours.

As you said [*] you'd love to have this feature in NetBSD 9.0, no 
objection neither. You still need some X86 specialist to review patch 3. 
The usual reviewers Paolo/Eduardo/Richard are currently very busy.

Also while I'd love to use this feature to be able to regularly run QEMU 
CI on NetBSD, I don't have time to test it on a bare metal hardware :|
Maybe do you know someone from the NetBSD community who already did?

[*] https://www.mail-archive.com/qemu-devel@nongnu.org/msg676199.html
Kamil Rytarowski Feb. 6, 2020, 4:59 p.m. UTC | #10
On 06.02.2020 17:07, Philippe Mathieu-Daudé wrote:
> On 2/6/20 4:38 PM, Kamil Rytarowski wrote:
>> On 06.02.2020 15:13, Markus Armbruster wrote:
>>> Kamil Rytarowski <n54@gmx.com> writes:
>>>
>>>> On 06.02.2020 14:09, Philippe Mathieu-Daudé wrote:
>>>>> On Thu, Feb 6, 2020 at 2:06 PM Markus Armbruster
>>>>> <armbru@redhat.com> wrote:
>>>>>> Kamil Rytarowski <n54@gmx.com> writes:
>>>>>>
>>>>>>> On 03.02.2020 12:54, Philippe Mathieu-Daudé wrote:
>>>>>>>>> @@ -2029,6 +2072,19 @@ static void
>>>>>>>>> qemu_whpx_start_vcpu(CPUState *cpu)
>>>>>>>>>    #endif
>>>>>>>>>    }
>>>>>>>>>
>>>>>>>>> +static void qemu_nvmm_start_vcpu(CPUState *cpu)
>>>>>>>>> +{
>>>>>>>>> +    char thread_name[VCPU_THREAD_NAME_SIZE];
>>>>>>>>> +
>>>>>>>>> +    cpu->thread = g_malloc0(sizeof(QemuThread));
>>>>>>>>> +    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
>>>>>>>>
>>>>>>>> Nitpick, we prefer g_new0().
>>>>>>>
>>>>>>> In this file other qemu_*_start_vcpu() use  g_malloc0().
>>>>>>>
>>>>>>> I will leave this part unchanged and defer tor future style
>>>>>>> fixups if
>>>>>>> someone is interested.
>>>>>>
>>>>>> Time to re-run Coccinelle with the semantic patch from commit
>>>>>> b45c03f585e.
>>>>>
>>>>> I thought about it, but then noticed it would be clever to modify
>>>>> checkpatch to refuse 'g_malloc0?(.*sizeof.*);'
>>>>>
>>>>>
>>>>
>>>> As the patchset was reviewed, could we please merge it in the current
>>>> (v3) form (*) please?
>>>
>>> No objection.  If I wanted you to clean this up before we accept your
>>> work, I would've told you :)
>>>
>>> [...]
>>>
>>>
>>
>> I see. I don't own myself a merge queue so I depend on yours.
>
> As you said [*] you'd love to have this feature in NetBSD 9.0, no
> objection neither. You still need some X86 specialist to review patch 3.
> The usual reviewers Paolo/Eduardo/Richard are currently very busy.
>
> Also while I'd love to use this feature to be able to regularly run QEMU
> CI on NetBSD, I don't have time to test it on a bare metal hardware :|
> Maybe do you know someone from the NetBSD community who already did?
>
> [*] https://www.mail-archive.com/qemu-devel@nongnu.org/msg676199.html
>
>

I'm going to find a person to test it and submit "Tested-by:".
diff mbox series

Patch

diff --git a/cpus.c b/cpus.c
index b472378b70..3c3f63588c 100644
--- a/cpus.c
+++ b/cpus.c
@@ -42,6 +42,7 @@ 
 #include "sysemu/hax.h"
 #include "sysemu/hvf.h"
 #include "sysemu/whpx.h"
+#include "sysemu/nvmm.h"
 #include "exec/exec-all.h"

 #include "qemu/thread.h"
@@ -1666,6 +1667,48 @@  static void *qemu_whpx_cpu_thread_fn(void *arg)
     return NULL;
 }

+static void *qemu_nvmm_cpu_thread_fn(void *arg)
+{
+    CPUState *cpu = arg;
+    int r;
+
+    assert(nvmm_enabled());
+
+    rcu_register_thread();
+
+    qemu_mutex_lock_iothread();
+    qemu_thread_get_self(cpu->thread);
+    cpu->thread_id = qemu_get_thread_id();
+    current_cpu = cpu;
+
+    r = nvmm_init_vcpu(cpu);
+    if (r < 0) {
+        fprintf(stderr, "nvmm_init_vcpu failed: %s\n", strerror(-r));
+        exit(1);
+    }
+
+    /* signal CPU creation */
+    cpu->created = true;
+    qemu_cond_signal(&qemu_cpu_cond);
+
+    do {
+        if (cpu_can_run(cpu)) {
+            r = nvmm_vcpu_exec(cpu);
+            if (r == EXCP_DEBUG) {
+                cpu_handle_guest_debug(cpu);
+            }
+        }
+        qemu_wait_io_event(cpu);
+    } while (!cpu->unplug || cpu_can_run(cpu));
+
+    nvmm_destroy_vcpu(cpu);
+    cpu->created = false;
+    qemu_cond_signal(&qemu_cpu_cond);
+    qemu_mutex_unlock_iothread();
+    rcu_unregister_thread();
+    return NULL;
+}
+
 #ifdef _WIN32
 static void CALLBACK dummy_apc_func(ULONG_PTR unused)
 {
@@ -2029,6 +2072,19 @@  static void qemu_whpx_start_vcpu(CPUState *cpu)
 #endif
 }

+static void qemu_nvmm_start_vcpu(CPUState *cpu)
+{
+    char thread_name[VCPU_THREAD_NAME_SIZE];
+
+    cpu->thread = g_malloc0(sizeof(QemuThread));
+    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+    qemu_cond_init(cpu->halt_cond);
+    snprintf(thread_name, VCPU_THREAD_NAME_SIZE, "CPU %d/NVMM",
+             cpu->cpu_index);
+    qemu_thread_create(cpu->thread, thread_name, qemu_nvmm_cpu_thread_fn,
+                       cpu, QEMU_THREAD_JOINABLE);
+}
+
 static void qemu_dummy_start_vcpu(CPUState *cpu)
 {
     char thread_name[VCPU_THREAD_NAME_SIZE];
@@ -2069,6 +2125,8 @@  void qemu_init_vcpu(CPUState *cpu)
         qemu_tcg_init_vcpu(cpu);
     } else if (whpx_enabled()) {
         qemu_whpx_start_vcpu(cpu);
+    } else if (nvmm_enabled()) {
+        qemu_nvmm_start_vcpu(cpu);
     } else {
         qemu_dummy_start_vcpu(cpu);
     }
diff --git a/include/sysemu/hw_accel.h b/include/sysemu/hw_accel.h
index 0ec2372477..dbfa7a02f9 100644
--- a/include/sysemu/hw_accel.h
+++ b/include/sysemu/hw_accel.h
@@ -15,6 +15,7 @@ 
 #include "sysemu/hax.h"
 #include "sysemu/kvm.h"
 #include "sysemu/whpx.h"
+#include "sysemu/nvmm.h"

 static inline void cpu_synchronize_state(CPUState *cpu)
 {
@@ -27,6 +28,9 @@  static inline void cpu_synchronize_state(CPUState *cpu)
     if (whpx_enabled()) {
         whpx_cpu_synchronize_state(cpu);
     }
+    if (nvmm_enabled()) {
+        nvmm_cpu_synchronize_state(cpu);
+    }
 }

 static inline void cpu_synchronize_post_reset(CPUState *cpu)
@@ -40,6 +44,10 @@  static inline void cpu_synchronize_post_reset(CPUState *cpu)
     if (whpx_enabled()) {
         whpx_cpu_synchronize_post_reset(cpu);
     }
+    if (nvmm_enabled()) {
+        nvmm_cpu_synchronize_post_reset(cpu);
+    }
+
 }

 static inline void cpu_synchronize_post_init(CPUState *cpu)
@@ -53,6 +61,9 @@  static inline void cpu_synchronize_post_init(CPUState *cpu)
     if (whpx_enabled()) {
         whpx_cpu_synchronize_post_init(cpu);
     }
+    if (nvmm_enabled()) {
+        nvmm_cpu_synchronize_post_init(cpu);
+    }
 }

 static inline void cpu_synchronize_pre_loadvm(CPUState *cpu)
@@ -66,6 +77,9 @@  static inline void cpu_synchronize_pre_loadvm(CPUState *cpu)
     if (whpx_enabled()) {
         whpx_cpu_synchronize_pre_loadvm(cpu);
     }
+    if (nvmm_enabled()) {
+        nvmm_cpu_synchronize_pre_loadvm(cpu);
+    }
 }

 #endif /* QEMU_HW_ACCEL_H */
diff --git a/target/i386/helper.c b/target/i386/helper.c
index c3a6e4fabe..2e79d61329 100644
--- a/target/i386/helper.c
+++ b/target/i386/helper.c
@@ -981,7 +981,7 @@  void cpu_report_tpr_access(CPUX86State *env, TPRAccess access)
     X86CPU *cpu = env_archcpu(env);
     CPUState *cs = env_cpu(env);

-    if (kvm_enabled() || whpx_enabled()) {
+    if (kvm_enabled() || whpx_enabled() || nvmm_enabled()) {
         env->tpr_access_type = access;

         cpu_interrupt(cs, CPU_INTERRUPT_TPR);