| Message ID | 19703.38782.929586.869640@gargle.gargle.HOWL |
|---|---|
| State | New |
| Headers | show |
On Thu, Dec 02, 2010 at 08:56:30PM +0800, Jason Wang wrote: > Michael S. Tsirkin writes: > > On Wed, Dec 01, 2010 at 01:45:09PM +0800, Jason Wang wrote: > > > Michael S. Tsirkin writes: > > > > Avoid sending out packets, and modifying > > > > device state, when VM is stopped. > > > > Add assert statements to verify this does not happen. > > > > > > > > Avoid scheduling bh when vhost-net is started. > > > > > > > > Stop bh when driver disabled bus mastering > > > > (we must not access memory after this). > > > > > > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com> > > > > > > > > > > There's no need to disable it bh we call qemu_aio_flush() after > > > vm_state_notify() in do_vm_stop(). And for timer, looks like every device should > > > stop its timer in vm state change handler, not only for virtio-net? > > > > BTW I fixed some typos. Here a fixed version. > > Jason, could you review/test please? > > > > Have done the test, it's more stable than before but still get small deltas in > cpu section. I didn't find any interesting difference by checking the > CPUX86State in the dest in kvm_arch_load_regs(), any thought on this? So which offsets are different? > BTW, looks like the error_code was missed in saving the cpu state: Post this as a separate patch please. > diff --git a/target-i386/cpu.h b/target-i386/cpu.h > index 35a1a51..145bb38 100644 > --- a/target-i386/cpu.h > +++ b/target-i386/cpu.h > @@ -687,7 +687,7 @@ typedef struct CPUX86State { > uint64_t pat; > > /* exception/interrupt handling */ > - int error_code; > + uint32_t error_code; > int exception_is_int; > target_ulong exception_next_eip; > target_ulong dr[8]; /* debug registers */ > @@ -935,7 +935,7 @@ CPUState *pc_new_cpu(const char *cpu_model); > #define cpu_list_id x86_cpu_list > #define cpudef_setup x86_cpudef_setup > > -#define CPU_SAVE_VERSION 12 > +#define CPU_SAVE_VERSION 13 > > /* MMU modes definitions */ > #define MMU_MODE0_SUFFIX _kernel > diff --git a/target-i386/machine.c b/target-i386/machine.c > index 4398801..fa231d8 100644 > --- a/target-i386/machine.c > +++ b/target-i386/machine.c > @@ -474,6 +474,8 @@ static const VMStateDescription vmstate_cpu = { > VMSTATE_UINT64_V(xcr0, CPUState, 12), > VMSTATE_UINT64_V(xstate_bv, CPUState, 12), > VMSTATE_YMMH_REGS_VARS(ymmh_regs, CPUState, CPU_NB_REGS, 12), > + > + VMSTATE_UINT32_V(error_code, CPUState, 13), > VMSTATE_END_OF_LIST() > /* The above list is not sorted /wrt version numbers, watch out! */ > }
On Thu, Dec 02, 2010 at 08:56:30PM +0800, Jason Wang wrote: > Michael S. Tsirkin writes: > > On Wed, Dec 01, 2010 at 01:45:09PM +0800, Jason Wang wrote: > > > Michael S. Tsirkin writes: > > > > Avoid sending out packets, and modifying > > > > device state, when VM is stopped. > > > > Add assert statements to verify this does not happen. > > > > > > > > Avoid scheduling bh when vhost-net is started. > > > > > > > > Stop bh when driver disabled bus mastering > > > > (we must not access memory after this). > > > > > > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com> > > > > > > > > > > There's no need to disable it bh we call qemu_aio_flush() after > > > vm_state_notify() in do_vm_stop(). And for timer, looks like every device should > > > stop its timer in vm state change handler, not only for virtio-net? > > > > BTW I fixed some typos. Here a fixed version. > > Jason, could you review/test please? > > > > Have done the test, it's more stable than before but still get small deltas in > cpu section. And just to clarify: no more deltas in the memory section? > I didn't find any interesting difference by checking the > CPUX86State in the dest in kvm_arch_load_regs(), any thought on this? > > BTW, looks like the error_code was missed in saving the cpu state: > > diff --git a/target-i386/cpu.h b/target-i386/cpu.h > index 35a1a51..145bb38 100644 > --- a/target-i386/cpu.h > +++ b/target-i386/cpu.h > @@ -687,7 +687,7 @@ typedef struct CPUX86State { > uint64_t pat; > > /* exception/interrupt handling */ > - int error_code; > + uint32_t error_code; > int exception_is_int; > target_ulong exception_next_eip; > target_ulong dr[8]; /* debug registers */ > @@ -935,7 +935,7 @@ CPUState *pc_new_cpu(const char *cpu_model); > #define cpu_list_id x86_cpu_list > #define cpudef_setup x86_cpudef_setup > > -#define CPU_SAVE_VERSION 12 > +#define CPU_SAVE_VERSION 13 > > /* MMU modes definitions */ > #define MMU_MODE0_SUFFIX _kernel > diff --git a/target-i386/machine.c b/target-i386/machine.c > index 4398801..fa231d8 100644 > --- a/target-i386/machine.c > +++ b/target-i386/machine.c > @@ -474,6 +474,8 @@ static const VMStateDescription vmstate_cpu = { > VMSTATE_UINT64_V(xcr0, CPUState, 12), > VMSTATE_UINT64_V(xstate_bv, CPUState, 12), > VMSTATE_YMMH_REGS_VARS(ymmh_regs, CPUState, CPU_NB_REGS, 12), > + > + VMSTATE_UINT32_V(error_code, CPUState, 13), > VMSTATE_END_OF_LIST() > /* The above list is not sorted /wrt version numbers, watch out! */ > }
Michael S. Tsirkin writes: > On Thu, Dec 02, 2010 at 08:56:30PM +0800, Jason Wang wrote: > > Michael S. Tsirkin writes: > > > On Wed, Dec 01, 2010 at 01:45:09PM +0800, Jason Wang wrote: > > > > Michael S. Tsirkin writes: > > > > > Avoid sending out packets, and modifying > > > > > device state, when VM is stopped. > > > > > Add assert statements to verify this does not happen. > > > > > > > > > > Avoid scheduling bh when vhost-net is started. > > > > > > > > > > Stop bh when driver disabled bus mastering > > > > > (we must not access memory after this). > > > > > > > > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com> > > > > > > > > > > > > > There's no need to disable it bh we call qemu_aio_flush() after > > > > vm_state_notify() in do_vm_stop(). And for timer, looks like every device should > > > > stop its timer in vm state change handler, not only for virtio-net? > > > > > > BTW I fixed some typos. Here a fixed version. > > > Jason, could you review/test please? > > > > > > > Have done the test, it's more stable than before but still get small deltas in > > cpu section. > > And just to clarify: no more deltas in the memory section? > Yes. And the offset for cpu section is 1161-1165 and sometimes I get deltas for ide section at offset 295 and 314. > > I didn't find any interesting difference by checking the > > CPUX86State in the dest in kvm_arch_load_regs(), any thought on this? > > > > BTW, looks like the error_code was missed in saving the cpu state: > > > > diff --git a/target-i386/cpu.h b/target-i386/cpu.h > > index 35a1a51..145bb38 100644 > > --- a/target-i386/cpu.h > > +++ b/target-i386/cpu.h > > @@ -687,7 +687,7 @@ typedef struct CPUX86State { > > uint64_t pat; > > > > /* exception/interrupt handling */ > > - int error_code; > > + uint32_t error_code; > > int exception_is_int; > > target_ulong exception_next_eip; > > target_ulong dr[8]; /* debug registers */ > > @@ -935,7 +935,7 @@ CPUState *pc_new_cpu(const char *cpu_model); > > #define cpu_list_id x86_cpu_list > > #define cpudef_setup x86_cpudef_setup > > > > -#define CPU_SAVE_VERSION 12 > > +#define CPU_SAVE_VERSION 13 > > > > /* MMU modes definitions */ > > #define MMU_MODE0_SUFFIX _kernel > > diff --git a/target-i386/machine.c b/target-i386/machine.c > > index 4398801..fa231d8 100644 > > --- a/target-i386/machine.c > > +++ b/target-i386/machine.c > > @@ -474,6 +474,8 @@ static const VMStateDescription vmstate_cpu = { > > VMSTATE_UINT64_V(xcr0, CPUState, 12), > > VMSTATE_UINT64_V(xstate_bv, CPUState, 12), > > VMSTATE_YMMH_REGS_VARS(ymmh_regs, CPUState, CPU_NB_REGS, 12), > > + > > + VMSTATE_UINT32_V(error_code, CPUState, 13), > > VMSTATE_END_OF_LIST() > > /* The above list is not sorted /wrt version numbers, watch out! */ > > }
On Thu, Dec 02, 2010 at 10:19:55PM +0800, Jason Wang wrote: > Michael S. Tsirkin writes: > > On Thu, Dec 02, 2010 at 08:56:30PM +0800, Jason Wang wrote: > > > Michael S. Tsirkin writes: > > > > On Wed, Dec 01, 2010 at 01:45:09PM +0800, Jason Wang wrote: > > > > > Michael S. Tsirkin writes: > > > > > > Avoid sending out packets, and modifying > > > > > > device state, when VM is stopped. > > > > > > Add assert statements to verify this does not happen. > > > > > > > > > > > > Avoid scheduling bh when vhost-net is started. > > > > > > > > > > > > Stop bh when driver disabled bus mastering > > > > > > (we must not access memory after this). > > > > > > > > > > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com> > > > > > > > > > > > > > > > > There's no need to disable it bh we call qemu_aio_flush() after > > > > > vm_state_notify() in do_vm_stop(). And for timer, looks like every device should > > > > > stop its timer in vm state change handler, not only for virtio-net? > > > > > > > > BTW I fixed some typos. Here a fixed version. > > > > Jason, could you review/test please? > > > > > > > > > > Have done the test, it's more stable than before but still get small deltas in > > > cpu section. > > > > And just to clarify: no more deltas in the memory section? > > > > Yes. > > And the offset for cpu section is 1161-1165 As far as I can say the state is in target-i386/machine.c static const VMStateDescription vmstate_cpu. Need to do some math to find this: I think this is mtrr_var, but maybe my math is off. I would sugest printing out the state and see what is changed exactly. > and sometimes I get deltas for ide > section at offset 295 and 314. I see that ide has some bh processing. Most likely that starts io after vmstop? I suggest adding a vm state handler and checking vm status in ide_dma_restart_bh. Start with an assert, just for debug. Also, what if we use virtio-blk? > > > I didn't find any interesting difference by checking the > > > CPUX86State in the dest in kvm_arch_load_regs(), any thought on this? > > > > > > BTW, looks like the error_code was missed in saving the cpu state: > > > > > > diff --git a/target-i386/cpu.h b/target-i386/cpu.h > > > index 35a1a51..145bb38 100644 > > > --- a/target-i386/cpu.h > > > +++ b/target-i386/cpu.h > > > @@ -687,7 +687,7 @@ typedef struct CPUX86State { > > > uint64_t pat; > > > > > > /* exception/interrupt handling */ > > > - int error_code; > > > + uint32_t error_code; > > > int exception_is_int; > > > target_ulong exception_next_eip; > > > target_ulong dr[8]; /* debug registers */ > > > @@ -935,7 +935,7 @@ CPUState *pc_new_cpu(const char *cpu_model); > > > #define cpu_list_id x86_cpu_list > > > #define cpudef_setup x86_cpudef_setup > > > > > > -#define CPU_SAVE_VERSION 12 > > > +#define CPU_SAVE_VERSION 13 > > > > > > /* MMU modes definitions */ > > > #define MMU_MODE0_SUFFIX _kernel > > > diff --git a/target-i386/machine.c b/target-i386/machine.c > > > index 4398801..fa231d8 100644 > > > --- a/target-i386/machine.c > > > +++ b/target-i386/machine.c > > > @@ -474,6 +474,8 @@ static const VMStateDescription vmstate_cpu = { > > > VMSTATE_UINT64_V(xcr0, CPUState, 12), > > > VMSTATE_UINT64_V(xstate_bv, CPUState, 12), > > > VMSTATE_YMMH_REGS_VARS(ymmh_regs, CPUState, CPU_NB_REGS, 12), > > > + > > > + VMSTATE_UINT32_V(error_code, CPUState, 13), > > > VMSTATE_END_OF_LIST() > > > /* The above list is not sorted /wrt version numbers, watch out! */ > > > }
Michael S. Tsirkin writes: > On Thu, Dec 02, 2010 at 10:19:55PM +0800, Jason Wang wrote: > > Michael S. Tsirkin writes: > > > On Thu, Dec 02, 2010 at 08:56:30PM +0800, Jason Wang wrote: > > > > Michael S. Tsirkin writes: > > > > > On Wed, Dec 01, 2010 at 01:45:09PM +0800, Jason Wang wrote: > > > > > > Michael S. Tsirkin writes: > > > > > > > Avoid sending out packets, and modifying > > > > > > > device state, when VM is stopped. > > > > > > > Add assert statements to verify this does not happen. > > > > > > > > > > > > > > Avoid scheduling bh when vhost-net is started. > > > > > > > > > > > > > > Stop bh when driver disabled bus mastering > > > > > > > (we must not access memory after this). > > > > > > > > > > > > > > Signed-off-by: Michael S. Tsirkin <mst@redhat.com> > > > > > > > > > > > > > > > > > > > There's no need to disable it bh we call qemu_aio_flush() after > > > > > > vm_state_notify() in do_vm_stop(). And for timer, looks like every device should > > > > > > stop its timer in vm state change handler, not only for virtio-net? > > > > > > > > > > BTW I fixed some typos. Here a fixed version. > > > > > Jason, could you review/test please? > > > > > > > > > > > > > Have done the test, it's more stable than before but still get small deltas in > > > > cpu section. > > > > > > And just to clarify: no more deltas in the memory section? > > > > > > > Yes. > > > > And the offset for cpu section is 1161-1165 > > As far as I can say the state is in > target-i386/machine.c > static const VMStateDescription vmstate_cpu. > Need to do some math to find this: > > I think this is mtrr_var, but maybe my math is off. > I would sugest printing out the state and see > what is changed exactly. > Try printing CPUX86State through gdb and the filed used to do the save/restore are the same. Have done the check for mtrr_var and the value are same for both src and dst. And looks like it was never used by kvm. > > > and sometimes I get deltas for ide > > section at offset 295 and 314. > > I see that ide has some bh processing. Most likely that starts io after > vmstop? I suggest adding a vm state handler and checking vm status in > ide_dma_restart_bh. > > Start with an assert, just for debug. > > Also, what if we use virtio-blk? > One byte delta for virtio-blk section at offset 377. And also get delta for ide section ( so I didn't try your patch of stopping bh of ide becuse for virtio-blk we even do not use ide ). > > > > > I didn't find any interesting difference by checking the > > > > CPUX86State in the dest in kvm_arch_load_regs(), any thought on this? > > > > > > > > BTW, looks like the error_code was missed in saving the cpu state: > > > > > > > > diff --git a/target-i386/cpu.h b/target-i386/cpu.h > > > > index 35a1a51..145bb38 100644 > > > > --- a/target-i386/cpu.h > > > > +++ b/target-i386/cpu.h > > > > @@ -687,7 +687,7 @@ typedef struct CPUX86State { > > > > uint64_t pat; > > > > > > > > /* exception/interrupt handling */ > > > > - int error_code; > > > > + uint32_t error_code; > > > > int exception_is_int; > > > > target_ulong exception_next_eip; > > > > target_ulong dr[8]; /* debug registers */ > > > > @@ -935,7 +935,7 @@ CPUState *pc_new_cpu(const char *cpu_model); > > > > #define cpu_list_id x86_cpu_list > > > > #define cpudef_setup x86_cpudef_setup > > > > > > > > -#define CPU_SAVE_VERSION 12 > > > > +#define CPU_SAVE_VERSION 13 > > > > > > > > /* MMU modes definitions */ > > > > #define MMU_MODE0_SUFFIX _kernel > > > > diff --git a/target-i386/machine.c b/target-i386/machine.c > > > > index 4398801..fa231d8 100644 > > > > --- a/target-i386/machine.c > > > > +++ b/target-i386/machine.c > > > > @@ -474,6 +474,8 @@ static const VMStateDescription vmstate_cpu = { > > > > VMSTATE_UINT64_V(xcr0, CPUState, 12), > > > > VMSTATE_UINT64_V(xstate_bv, CPUState, 12), > > > > VMSTATE_YMMH_REGS_VARS(ymmh_regs, CPUState, CPU_NB_REGS, 12), > > > > + > > > > + VMSTATE_UINT32_V(error_code, CPUState, 13), > > > > VMSTATE_END_OF_LIST() > > > > /* The above list is not sorted /wrt version numbers, watch out! */ > > > > }
diff --git a/target-i386/cpu.h b/target-i386/cpu.h index 35a1a51..145bb38 100644 --- a/target-i386/cpu.h +++ b/target-i386/cpu.h @@ -687,7 +687,7 @@ typedef struct CPUX86State { uint64_t pat; /* exception/interrupt handling */ - int error_code; + uint32_t error_code; int exception_is_int; target_ulong exception_next_eip; target_ulong dr[8]; /* debug registers */ @@ -935,7 +935,7 @@ CPUState *pc_new_cpu(const char *cpu_model); #define cpu_list_id x86_cpu_list #define cpudef_setup x86_cpudef_setup -#define CPU_SAVE_VERSION 12 +#define CPU_SAVE_VERSION 13 /* MMU modes definitions */ #define MMU_MODE0_SUFFIX _kernel diff --git a/target-i386/machine.c b/target-i386/machine.c index 4398801..fa231d8 100644 --- a/target-i386/machine.c +++ b/target-i386/machine.c @@ -474,6 +474,8 @@ static const VMStateDescription vmstate_cpu = { VMSTATE_UINT64_V(xcr0, CPUState, 12), VMSTATE_UINT64_V(xstate_bv, CPUState, 12), VMSTATE_YMMH_REGS_VARS(ymmh_regs, CPUState, CPU_NB_REGS, 12), + + VMSTATE_UINT32_V(error_code, CPUState, 13), VMSTATE_END_OF_LIST() /* The above list is not sorted /wrt version numbers, watch out! */ }