Message ID | 20231025193822.2813204-14-hao.xiang@bytedance.com |
---|---|
State | New |
Headers | show |
Series | Use Intel DSA accelerator to offload zero page checking in multifd live migration. | expand |
Hao Xiang <hao.xiang@bytedance.com> writes: > The current multifd packet size is 128 * 4kb. This change adds > an option to set the packet size. Both sender and receiver needs > to set the same packet size for things to work. > > Signed-off-by: Hao Xiang <hao.xiang@bytedance.com> > --- > migration/options.c | 34 ++++++++++++++++++++++++++++++++++ > migration/options.h | 1 + > qapi/migration.json | 20 +++++++++++++++++--- > 3 files changed, 52 insertions(+), 3 deletions(-) > > diff --git a/migration/options.c b/migration/options.c > index 9ee0ad5d89..6cb3d19470 100644 > --- a/migration/options.c > +++ b/migration/options.c > @@ -83,6 +83,12 @@ > */ > #define DEFAULT_MIGRATE_MULTIFD_NORMAL_PAGE_RATIO 101 > > +/* > + * Parameter for multifd packet size. > + */ > +#define DEFAULT_MIGRATE_MULTIFD_PACKET_SIZE (512 * 1024) > +#define MAX_MIGRATE_MULTIFD_PACKET_SIZE (1024 * 4 * 1024) > + > #define DEFINE_PROP_MIG_CAP(name, x) \ > DEFINE_PROP_BOOL(name, MigrationState, capabilities[x], false) > > @@ -183,6 +189,9 @@ Property migration_properties[] = { > DEFINE_PROP_UINT8("multifd-normal-page-ratio", MigrationState, > parameters.multifd_normal_page_ratio, > DEFAULT_MIGRATE_MULTIFD_NORMAL_PAGE_RATIO), > + DEFINE_PROP_SIZE("multifd-packet-size", MigrationState, > + parameters.multifd_packet_size, > + DEFAULT_MIGRATE_MULTIFD_PACKET_SIZE), > > /* Migration capabilities */ > DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE), > @@ -822,6 +831,13 @@ uint8_t migrate_multifd_normal_page_ratio(void) > return s->parameters.multifd_normal_page_ratio; > } > > +uint64_t migrate_multifd_packet_size(void) > +{ > + MigrationState *s = migrate_get_current(); > + > + return s->parameters.multifd_packet_size; > +} > + > MultiFDCompression migrate_multifd_compression(void) > { > MigrationState *s = migrate_get_current(); > @@ -958,6 +974,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) > params->x_checkpoint_delay = s->parameters.x_checkpoint_delay; > params->has_block_incremental = true; > params->block_incremental = s->parameters.block_incremental; > + params->has_multifd_packet_size = true; > + params->multifd_packet_size = s->parameters.multifd_packet_size; > params->has_multifd_channels = true; > params->multifd_channels = s->parameters.multifd_channels; > params->has_multifd_compression = true; > @@ -1016,6 +1034,7 @@ void migrate_params_init(MigrationParameters *params) > params->has_downtime_limit = true; > params->has_x_checkpoint_delay = true; > params->has_block_incremental = true; > + params->has_multifd_packet_size = true; > params->has_multifd_channels = true; > params->has_multifd_compression = true; > params->has_multifd_zlib_level = true; > @@ -1104,6 +1123,15 @@ bool migrate_params_check(MigrationParameters *params, Error **errp) > > /* x_checkpoint_delay is now always positive */ > > + if (params->has_multifd_packet_size && > + ((params->multifd_packet_size < DEFAULT_MIGRATE_MULTIFD_PACKET_SIZE) || > + (params->multifd_packet_size > MAX_MIGRATE_MULTIFD_PACKET_SIZE))) { > + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, > + "multifd_packet_size", > + "a value between 524288 and 4194304"); > + return false; > + } > + > if (params->has_multifd_channels && (params->multifd_channels < 1)) { > error_setg(errp, QERR_INVALID_PARAMETER_VALUE, > "multifd_channels", > @@ -1281,6 +1309,9 @@ static void migrate_params_test_apply(MigrateSetParameters *params, > if (params->has_block_incremental) { > dest->block_incremental = params->block_incremental; > } > + if (params->has_multifd_packet_size) { > + dest->multifd_packet_size = params->multifd_packet_size; > + } > if (params->has_multifd_channels) { > dest->multifd_channels = params->multifd_channels; > } > @@ -1408,6 +1439,9 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) > if (params->has_block_incremental) { > s->parameters.block_incremental = params->block_incremental; > } > + if (params->has_multifd_packet_size) { > + s->parameters.multifd_packet_size = params->multifd_packet_size; > + } > if (params->has_multifd_channels) { > s->parameters.multifd_channels = params->multifd_channels; > } > diff --git a/migration/options.h b/migration/options.h > index dafb09d6ea..1170971aef 100644 > --- a/migration/options.h > +++ b/migration/options.h > @@ -93,6 +93,7 @@ const char *migrate_tls_hostname(void); > uint64_t migrate_xbzrle_cache_size(void); > const char *migrate_multifd_dsa_accel(void); > uint8_t migrate_multifd_normal_page_ratio(void); > +uint64_t migrate_multifd_packet_size(void); > > /* parameters setters */ > > diff --git a/qapi/migration.json b/qapi/migration.json > index a667527081..a492b73060 100644 > --- a/qapi/migration.json > +++ b/qapi/migration.json > @@ -835,6 +835,10 @@ > # @multifd-normal-page-ratio: Test hook setting the normal page ratio. > # (Since 8.1) > # > +# @multifd-packet-size: Packet size used to migrate data. This value > +# needs to be a multiple of qemu_target_page_size(). The default > +# value is (512 * 1024) (Since 8.0) User doesn't know what qemu_target_page_size means. I'd also not have them do arithmetic, specially since the option takes a single number. We also need to enforce this in the code in the next patch. > +# > # Features: > # > # @unstable: Members @x-checkpoint-delay and @x-vcpu-dirty-limit-period > @@ -859,7 +863,7 @@ > 'multifd-zlib-level', 'multifd-zstd-level', > 'block-bitmap-mapping', > { 'name': 'x-vcpu-dirty-limit-period', 'features': ['unstable'] }, > - 'vcpu-dirty-limit', 'multifd-normal-page-ratio'] } > + 'vcpu-dirty-limit', 'multifd-normal-page-ratio', 'multifd-packet-size'] } > > ## > # @MigrateSetParameters: > @@ -1007,6 +1011,10 @@ > # @multifd-normal-page-ratio: Test hook setting the normal page ratio. > # (Since 8.1) > # > +# @multifd-packet-size: Packet size used to migrate data. This value > +# needs to be a multiple of qemu_target_page_size(). The default > +# value is (512 * 1024) (Since 8.0) > +# > # Features: > # > # @unstable: Members @x-checkpoint-delay and @x-vcpu-dirty-limit-period > @@ -1050,7 +1058,8 @@ > 'features': [ 'unstable' ] }, > '*vcpu-dirty-limit': 'uint64', > '*multifd-dsa-accel': 'StrOrNull', > - '*multifd-normal-page-ratio': 'uint8'} } > + '*multifd-normal-page-ratio': 'uint8', > + '*multifd-packet-size' : 'uint64'} } > > ## > # @migrate-set-parameters: > @@ -1218,6 +1227,10 @@ > # @multifd-normal-page-ratio: Test hook setting the normal page ratio. > # (Since 8.1) > # > +# @multifd-packet-size: Packet size used to migrate data. This value > +# needs to be a multiple of qemu_target_page_size(). The default > +# value is (512 * 1024) (Since 8.0) > +# > # Features: > # > # @unstable: Members @x-checkpoint-delay and @x-vcpu-dirty-limit-period > @@ -1258,7 +1271,8 @@ > 'features': [ 'unstable' ] }, > '*vcpu-dirty-limit': 'uint64', > '*multifd-dsa-accel': 'str', > - '*multifd-normal-page-ratio': 'uint8'} } > + '*multifd-normal-page-ratio': 'uint8', > + '*multifd-packet-size': 'uint64'} } > > ## > # @query-migrate-parameters:
diff --git a/migration/options.c b/migration/options.c index 9ee0ad5d89..6cb3d19470 100644 --- a/migration/options.c +++ b/migration/options.c @@ -83,6 +83,12 @@ */ #define DEFAULT_MIGRATE_MULTIFD_NORMAL_PAGE_RATIO 101 +/* + * Parameter for multifd packet size. + */ +#define DEFAULT_MIGRATE_MULTIFD_PACKET_SIZE (512 * 1024) +#define MAX_MIGRATE_MULTIFD_PACKET_SIZE (1024 * 4 * 1024) + #define DEFINE_PROP_MIG_CAP(name, x) \ DEFINE_PROP_BOOL(name, MigrationState, capabilities[x], false) @@ -183,6 +189,9 @@ Property migration_properties[] = { DEFINE_PROP_UINT8("multifd-normal-page-ratio", MigrationState, parameters.multifd_normal_page_ratio, DEFAULT_MIGRATE_MULTIFD_NORMAL_PAGE_RATIO), + DEFINE_PROP_SIZE("multifd-packet-size", MigrationState, + parameters.multifd_packet_size, + DEFAULT_MIGRATE_MULTIFD_PACKET_SIZE), /* Migration capabilities */ DEFINE_PROP_MIG_CAP("x-xbzrle", MIGRATION_CAPABILITY_XBZRLE), @@ -822,6 +831,13 @@ uint8_t migrate_multifd_normal_page_ratio(void) return s->parameters.multifd_normal_page_ratio; } +uint64_t migrate_multifd_packet_size(void) +{ + MigrationState *s = migrate_get_current(); + + return s->parameters.multifd_packet_size; +} + MultiFDCompression migrate_multifd_compression(void) { MigrationState *s = migrate_get_current(); @@ -958,6 +974,8 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) params->x_checkpoint_delay = s->parameters.x_checkpoint_delay; params->has_block_incremental = true; params->block_incremental = s->parameters.block_incremental; + params->has_multifd_packet_size = true; + params->multifd_packet_size = s->parameters.multifd_packet_size; params->has_multifd_channels = true; params->multifd_channels = s->parameters.multifd_channels; params->has_multifd_compression = true; @@ -1016,6 +1034,7 @@ void migrate_params_init(MigrationParameters *params) params->has_downtime_limit = true; params->has_x_checkpoint_delay = true; params->has_block_incremental = true; + params->has_multifd_packet_size = true; params->has_multifd_channels = true; params->has_multifd_compression = true; params->has_multifd_zlib_level = true; @@ -1104,6 +1123,15 @@ bool migrate_params_check(MigrationParameters *params, Error **errp) /* x_checkpoint_delay is now always positive */ + if (params->has_multifd_packet_size && + ((params->multifd_packet_size < DEFAULT_MIGRATE_MULTIFD_PACKET_SIZE) || + (params->multifd_packet_size > MAX_MIGRATE_MULTIFD_PACKET_SIZE))) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "multifd_packet_size", + "a value between 524288 and 4194304"); + return false; + } + if (params->has_multifd_channels && (params->multifd_channels < 1)) { error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_channels", @@ -1281,6 +1309,9 @@ static void migrate_params_test_apply(MigrateSetParameters *params, if (params->has_block_incremental) { dest->block_incremental = params->block_incremental; } + if (params->has_multifd_packet_size) { + dest->multifd_packet_size = params->multifd_packet_size; + } if (params->has_multifd_channels) { dest->multifd_channels = params->multifd_channels; } @@ -1408,6 +1439,9 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) if (params->has_block_incremental) { s->parameters.block_incremental = params->block_incremental; } + if (params->has_multifd_packet_size) { + s->parameters.multifd_packet_size = params->multifd_packet_size; + } if (params->has_multifd_channels) { s->parameters.multifd_channels = params->multifd_channels; } diff --git a/migration/options.h b/migration/options.h index dafb09d6ea..1170971aef 100644 --- a/migration/options.h +++ b/migration/options.h @@ -93,6 +93,7 @@ const char *migrate_tls_hostname(void); uint64_t migrate_xbzrle_cache_size(void); const char *migrate_multifd_dsa_accel(void); uint8_t migrate_multifd_normal_page_ratio(void); +uint64_t migrate_multifd_packet_size(void); /* parameters setters */ diff --git a/qapi/migration.json b/qapi/migration.json index a667527081..a492b73060 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -835,6 +835,10 @@ # @multifd-normal-page-ratio: Test hook setting the normal page ratio. # (Since 8.1) # +# @multifd-packet-size: Packet size used to migrate data. This value +# needs to be a multiple of qemu_target_page_size(). The default +# value is (512 * 1024) (Since 8.0) +# # Features: # # @unstable: Members @x-checkpoint-delay and @x-vcpu-dirty-limit-period @@ -859,7 +863,7 @@ 'multifd-zlib-level', 'multifd-zstd-level', 'block-bitmap-mapping', { 'name': 'x-vcpu-dirty-limit-period', 'features': ['unstable'] }, - 'vcpu-dirty-limit', 'multifd-normal-page-ratio'] } + 'vcpu-dirty-limit', 'multifd-normal-page-ratio', 'multifd-packet-size'] } ## # @MigrateSetParameters: @@ -1007,6 +1011,10 @@ # @multifd-normal-page-ratio: Test hook setting the normal page ratio. # (Since 8.1) # +# @multifd-packet-size: Packet size used to migrate data. This value +# needs to be a multiple of qemu_target_page_size(). The default +# value is (512 * 1024) (Since 8.0) +# # Features: # # @unstable: Members @x-checkpoint-delay and @x-vcpu-dirty-limit-period @@ -1050,7 +1058,8 @@ 'features': [ 'unstable' ] }, '*vcpu-dirty-limit': 'uint64', '*multifd-dsa-accel': 'StrOrNull', - '*multifd-normal-page-ratio': 'uint8'} } + '*multifd-normal-page-ratio': 'uint8', + '*multifd-packet-size' : 'uint64'} } ## # @migrate-set-parameters: @@ -1218,6 +1227,10 @@ # @multifd-normal-page-ratio: Test hook setting the normal page ratio. # (Since 8.1) # +# @multifd-packet-size: Packet size used to migrate data. This value +# needs to be a multiple of qemu_target_page_size(). The default +# value is (512 * 1024) (Since 8.0) +# # Features: # # @unstable: Members @x-checkpoint-delay and @x-vcpu-dirty-limit-period @@ -1258,7 +1271,8 @@ 'features': [ 'unstable' ] }, '*vcpu-dirty-limit': 'uint64', '*multifd-dsa-accel': 'str', - '*multifd-normal-page-ratio': 'uint8'} } + '*multifd-normal-page-ratio': 'uint8', + '*multifd-packet-size': 'uint64'} } ## # @query-migrate-parameters:
The current multifd packet size is 128 * 4kb. This change adds an option to set the packet size. Both sender and receiver needs to set the same packet size for things to work. Signed-off-by: Hao Xiang <hao.xiang@bytedance.com> --- migration/options.c | 34 ++++++++++++++++++++++++++++++++++ migration/options.h | 1 + qapi/migration.json | 20 +++++++++++++++++--- 3 files changed, 52 insertions(+), 3 deletions(-)