| Message ID | 20230628073112.7958-2-avihaih@nvidia.com |
|---|---|
| State | New |
| Headers | show |
| Series | vfio/migration: Make VFIO migration non-experimental | expand |
On 6/28/23 09:31, Avihai Horon wrote: > Currently, VFIO bytes_transferred is not reset properly: > 1. bytes_transferred is not reset after a VM snapshot (so a migration > following a snapshot will report incorrect value). > 2. bytes_transferred is a single counter for all VFIO devices, however > upon migration failure it is reset multiple times, by each VFIO > device. > > Fix it by introducing a new function vfio_reset_bytes_transferred() and > calling it during migration and snapshot start. > > Remove existing bytes_transferred reset in VFIO migration state > notifier, which is not needed anymore. > > Fixes: 3710586caa5d ("qapi: Add VFIO devices migration stats in Migration stats") > Signed-off-by: Avihai Horon <avihaih@nvidia.com> Reviewed-by: Cédric Le Goater <clg@redhat.com> Thanks, C. > --- > include/hw/vfio/vfio-common.h | 1 + > migration/migration.h | 1 + > hw/vfio/migration.c | 6 +++++- > migration/migration.c | 1 + > migration/savevm.c | 1 + > migration/target.c | 17 +++++++++++++++-- > 6 files changed, 24 insertions(+), 3 deletions(-) > > diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h > index 3dc5f2104c..b4c28f318f 100644 > --- a/include/hw/vfio/vfio-common.h > +++ b/include/hw/vfio/vfio-common.h > @@ -228,6 +228,7 @@ int vfio_block_multiple_devices_migration(Error **errp); > void vfio_unblock_multiple_devices_migration(void); > int vfio_block_giommu_migration(Error **errp); > int64_t vfio_mig_bytes_transferred(void); > +void vfio_reset_bytes_transferred(void); > > #ifdef CONFIG_LINUX > int vfio_get_region_info(VFIODevice *vbasedev, int index, > diff --git a/migration/migration.h b/migration/migration.h > index c859a0d35e..a80b22b703 100644 > --- a/migration/migration.h > +++ b/migration/migration.h > @@ -514,6 +514,7 @@ bool migration_rate_limit(void); > void migration_cancel(const Error *error); > > void populate_vfio_info(MigrationInfo *info); > +void reset_vfio_bytes_transferred(void); > void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page); > > #endif > diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c > index acbf0bb7ab..7cf143926c 100644 > --- a/hw/vfio/migration.c > +++ b/hw/vfio/migration.c > @@ -697,7 +697,6 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data) > case MIGRATION_STATUS_CANCELLING: > case MIGRATION_STATUS_CANCELLED: > case MIGRATION_STATUS_FAILED: > - bytes_transferred = 0; > /* > * If setting the device in RUNNING state fails, the device should > * be reset. To do so, use ERROR state as a recover state. > @@ -818,6 +817,11 @@ int64_t vfio_mig_bytes_transferred(void) > return bytes_transferred; > } > > +void vfio_reset_bytes_transferred(void) > +{ > + bytes_transferred = 0; > +} > + > int vfio_migration_realize(VFIODevice *vbasedev, Error **errp) > { > int ret = -ENOTSUP; > diff --git a/migration/migration.c b/migration/migration.c > index 7653787f74..096e8191d1 100644 > --- a/migration/migration.c > +++ b/migration/migration.c > @@ -1628,6 +1628,7 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc, > */ > memset(&mig_stats, 0, sizeof(mig_stats)); > memset(&compression_counters, 0, sizeof(compression_counters)); > + reset_vfio_bytes_transferred(); > > return true; > } > diff --git a/migration/savevm.c b/migration/savevm.c > index cdf4793924..95c2abf47c 100644 > --- a/migration/savevm.c > +++ b/migration/savevm.c > @@ -1622,6 +1622,7 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp) > migrate_init(ms); > memset(&mig_stats, 0, sizeof(mig_stats)); > memset(&compression_counters, 0, sizeof(compression_counters)); > + reset_vfio_bytes_transferred(); > ms->to_dst_file = f; > > qemu_mutex_unlock_iothread(); > diff --git a/migration/target.c b/migration/target.c > index 00ca007f97..f39c9a8d88 100644 > --- a/migration/target.c > +++ b/migration/target.c > @@ -14,12 +14,25 @@ > #include "hw/vfio/vfio-common.h" > #endif > > +#ifdef CONFIG_VFIO > void populate_vfio_info(MigrationInfo *info) > { > -#ifdef CONFIG_VFIO > if (vfio_mig_active()) { > info->vfio = g_malloc0(sizeof(*info->vfio)); > info->vfio->transferred = vfio_mig_bytes_transferred(); > } > -#endif > } > + > +void reset_vfio_bytes_transferred(void) > +{ > + vfio_reset_bytes_transferred(); > +} > +#else > +void populate_vfio_info(MigrationInfo *info) > +{ > +} > + > +void reset_vfio_bytes_transferred(void) > +{ > +} > +#endif
diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h index 3dc5f2104c..b4c28f318f 100644 --- a/include/hw/vfio/vfio-common.h +++ b/include/hw/vfio/vfio-common.h @@ -228,6 +228,7 @@ int vfio_block_multiple_devices_migration(Error **errp); void vfio_unblock_multiple_devices_migration(void); int vfio_block_giommu_migration(Error **errp); int64_t vfio_mig_bytes_transferred(void); +void vfio_reset_bytes_transferred(void); #ifdef CONFIG_LINUX int vfio_get_region_info(VFIODevice *vbasedev, int index, diff --git a/migration/migration.h b/migration/migration.h index c859a0d35e..a80b22b703 100644 --- a/migration/migration.h +++ b/migration/migration.h @@ -514,6 +514,7 @@ bool migration_rate_limit(void); void migration_cancel(const Error *error); void populate_vfio_info(MigrationInfo *info); +void reset_vfio_bytes_transferred(void); void postcopy_temp_page_reset(PostcopyTmpPage *tmp_page); #endif diff --git a/hw/vfio/migration.c b/hw/vfio/migration.c index acbf0bb7ab..7cf143926c 100644 --- a/hw/vfio/migration.c +++ b/hw/vfio/migration.c @@ -697,7 +697,6 @@ static void vfio_migration_state_notifier(Notifier *notifier, void *data) case MIGRATION_STATUS_CANCELLING: case MIGRATION_STATUS_CANCELLED: case MIGRATION_STATUS_FAILED: - bytes_transferred = 0; /* * If setting the device in RUNNING state fails, the device should * be reset. To do so, use ERROR state as a recover state. @@ -818,6 +817,11 @@ int64_t vfio_mig_bytes_transferred(void) return bytes_transferred; } +void vfio_reset_bytes_transferred(void) +{ + bytes_transferred = 0; +} + int vfio_migration_realize(VFIODevice *vbasedev, Error **errp) { int ret = -ENOTSUP; diff --git a/migration/migration.c b/migration/migration.c index 7653787f74..096e8191d1 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -1628,6 +1628,7 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc, */ memset(&mig_stats, 0, sizeof(mig_stats)); memset(&compression_counters, 0, sizeof(compression_counters)); + reset_vfio_bytes_transferred(); return true; } diff --git a/migration/savevm.c b/migration/savevm.c index cdf4793924..95c2abf47c 100644 --- a/migration/savevm.c +++ b/migration/savevm.c @@ -1622,6 +1622,7 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp) migrate_init(ms); memset(&mig_stats, 0, sizeof(mig_stats)); memset(&compression_counters, 0, sizeof(compression_counters)); + reset_vfio_bytes_transferred(); ms->to_dst_file = f; qemu_mutex_unlock_iothread(); diff --git a/migration/target.c b/migration/target.c index 00ca007f97..f39c9a8d88 100644 --- a/migration/target.c +++ b/migration/target.c @@ -14,12 +14,25 @@ #include "hw/vfio/vfio-common.h" #endif +#ifdef CONFIG_VFIO void populate_vfio_info(MigrationInfo *info) { -#ifdef CONFIG_VFIO if (vfio_mig_active()) { info->vfio = g_malloc0(sizeof(*info->vfio)); info->vfio->transferred = vfio_mig_bytes_transferred(); } -#endif } + +void reset_vfio_bytes_transferred(void) +{ + vfio_reset_bytes_transferred(); +} +#else +void populate_vfio_info(MigrationInfo *info) +{ +} + +void reset_vfio_bytes_transferred(void) +{ +} +#endif
Currently, VFIO bytes_transferred is not reset properly: 1. bytes_transferred is not reset after a VM snapshot (so a migration following a snapshot will report incorrect value). 2. bytes_transferred is a single counter for all VFIO devices, however upon migration failure it is reset multiple times, by each VFIO device. Fix it by introducing a new function vfio_reset_bytes_transferred() and calling it during migration and snapshot start. Remove existing bytes_transferred reset in VFIO migration state notifier, which is not needed anymore. Fixes: 3710586caa5d ("qapi: Add VFIO devices migration stats in Migration stats") Signed-off-by: Avihai Horon <avihaih@nvidia.com> --- include/hw/vfio/vfio-common.h | 1 + migration/migration.h | 1 + hw/vfio/migration.c | 6 +++++- migration/migration.c | 1 + migration/savevm.c | 1 + migration/target.c | 17 +++++++++++++++-- 6 files changed, 24 insertions(+), 3 deletions(-)