| Message ID | 20240326224221.3623014-3-bryan.zhang@bytedance.com |
|---|---|
| State | New |
| Headers | show |
| Series | *** Implement using Intel QAT to offload ZLIB | expand |
> -----Original Message----- > From: Bryan Zhang <bryan.zhang@bytedance.com> > Sent: Wednesday, March 27, 2024 6:42 AM > To: qemu-devel@nongnu.org > Cc: peterx@redhat.com; farosas@suse.de; Liu, Yuan1 <yuan1.liu@intel.com>; > berrange@redhat.com; Zou, Nanhai <nanhai.zou@intel.com>; > hao.xiang@linux.dev; Bryan Zhang <bryan.zhang@bytedance.com> > Subject: [PATCH v2 2/5] migration: Add migration parameters for QATzip > > Adds support for migration parameters to control QATzip compression > level and to enable/disable software fallback when QAT hardware is > unavailable. This is a preparatory commit for a subsequent commit that > will actually use QATzip compression. > > Signed-off-by: Bryan Zhang <bryan.zhang@bytedance.com> > Signed-off-by: Hao Xiang <hao.xiang@linux.dev> > --- > Revision: This commit now includes a parameter for controlling software > fallback. Fallback is generally intended to be disabled, but having this > option available enables using software fallback for testing. > > This commit also now has some glue code to properly set parameters. > > migration/migration-hmp-cmds.c | 8 +++++ > migration/options.c | 57 ++++++++++++++++++++++++++++++++++ > migration/options.h | 2 ++ > qapi/migration.json | 35 +++++++++++++++++++++ > 4 files changed, 102 insertions(+) > > diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp- > cmds.c > index 99b49df5dd..4bd23ba14d 100644 > --- a/migration/migration-hmp-cmds.c > +++ b/migration/migration-hmp-cmds.c > @@ -630,6 +630,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const > QDict *qdict) > p->has_multifd_zlib_level = true; > visit_type_uint8(v, param, &p->multifd_zlib_level, &err); > break; > + case MIGRATION_PARAMETER_MULTIFD_QATZIP_LEVEL: > + p->has_multifd_qatzip_level = true; > + visit_type_uint8(v, param, &p->multifd_qatzip_level, &err); > + break; > + case MIGRATION_PARAMETER_MULTIFD_QATZIP_SW_FALLBACK: > + p->has_multifd_qatzip_sw_fallback = true; > + visit_type_bool(v, param, &p->multifd_qatzip_sw_fallback, &err); > + break; > case MIGRATION_PARAMETER_MULTIFD_ZSTD_LEVEL: > p->has_multifd_zstd_level = true; > visit_type_uint8(v, param, &p->multifd_zstd_level, &err); > diff --git a/migration/options.c b/migration/options.c > index 3e3e0b93b4..1316ea605a 100644 > --- a/migration/options.c > +++ b/migration/options.c > @@ -62,6 +62,15 @@ > #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE > /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */ > #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1 > +/* > + * 1: best speed, ... 9: best compress ratio > + * There is some nuance here. Refer to QATzip documentation to understand > + * the mapping of QATzip levels to standard deflate levels. > + */ > +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL 1 > +/* QATzip's SW fallback implementation is extremely slow, so avoid > fallback */ > +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK false > + > /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */ > #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1 Hi Bryan The default compression level may be set higher, such as 6. I checked QAT throughput If the data size is less than or equal to 64K, level 1 has much better throughput performance than level 6 and level 9. But if the data size is greater than 128K, little change in throughput, and the default MULTIFD_PACKET_SIZE is 512K, you can have a try to use a high compression level, to get better compression performance without affecting throughput. In addition, if you change MULTIFD_PACKET_SIZE to 64K, you may have better throughput with more multifd threads > @@ -143,6 +152,12 @@ Property migration_properties[] = { > DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState, > parameters.multifd_zlib_level, > DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL), > + DEFINE_PROP_UINT8("multifd-qatzip-level", MigrationState, > + parameters.multifd_qatzip_level, > + DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL), > + DEFINE_PROP_BOOL("multifd-qatzip-sw-fallback", MigrationState, > + parameters.multifd_qatzip_sw_fallback, > + DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK), > DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState, > parameters.multifd_zstd_level, > DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL), > @@ -861,6 +876,20 @@ int migrate_multifd_zlib_level(void) > return s->parameters.multifd_zlib_level; > } > > +int migrate_multifd_qatzip_level(void) > +{ > + MigrationState *s = migrate_get_current(); > + > + return s->parameters.multifd_qatzip_level; > +} > + > +bool migrate_multifd_qatzip_sw_fallback(void) > +{ > + MigrationState *s = migrate_get_current(); > + > + return s->parameters.multifd_qatzip_sw_fallback; > +} > + > int migrate_multifd_zstd_level(void) > { > MigrationState *s = migrate_get_current(); > @@ -983,6 +1012,11 @@ MigrationParameters > *qmp_query_migrate_parameters(Error **errp) > params->multifd_compression = s->parameters.multifd_compression; > params->has_multifd_zlib_level = true; > params->multifd_zlib_level = s->parameters.multifd_zlib_level; > + params->has_multifd_qatzip_level = true; > + params->multifd_qatzip_level = s->parameters.multifd_qatzip_level; > + params->has_multifd_qatzip_sw_fallback = true; > + params->multifd_qatzip_sw_fallback = > + s->parameters.multifd_qatzip_sw_fallback; > params->has_multifd_zstd_level = true; > params->multifd_zstd_level = s->parameters.multifd_zstd_level; > params->has_xbzrle_cache_size = true; > @@ -1038,6 +1072,8 @@ void migrate_params_init(MigrationParameters > *params) > params->has_multifd_channels = true; > params->has_multifd_compression = true; > params->has_multifd_zlib_level = true; > + params->has_multifd_qatzip_level = true; > + params->has_multifd_qatzip_sw_fallback = true; > params->has_multifd_zstd_level = true; > params->has_xbzrle_cache_size = true; > params->has_max_postcopy_bandwidth = true; > @@ -1147,6 +1183,14 @@ bool migrate_params_check(MigrationParameters > *params, Error **errp) > return false; > } > > + if (params->has_multifd_qatzip_level && > + ((params->multifd_qatzip_level > 9) || > + (params->multifd_qatzip_level < 1))) { > + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, > "multifd_qatzip_level", > + "a value between 1 and 9"); > + return false; > + } > + > if (params->has_multifd_zstd_level && > (params->multifd_zstd_level > 20)) { > error_setg(errp, QERR_INVALID_PARAMETER_VALUE, > "multifd_zstd_level", > @@ -1312,6 +1356,12 @@ static void > migrate_params_test_apply(MigrateSetParameters *params, > if (params->has_multifd_compression) { > dest->multifd_compression = params->multifd_compression; > } > + if (params->has_multifd_qatzip_level) { > + dest->multifd_qatzip_level = params->multifd_qatzip_level; > + } > + if (params->has_multifd_qatzip_sw_fallback) { > + dest->multifd_qatzip_sw_fallback = params- > >multifd_qatzip_sw_fallback; > + } > if (params->has_xbzrle_cache_size) { > dest->xbzrle_cache_size = params->xbzrle_cache_size; > } > @@ -1447,6 +1497,13 @@ static void > migrate_params_apply(MigrateSetParameters *params, Error **errp) > if (params->has_multifd_compression) { > s->parameters.multifd_compression = params->multifd_compression; > } > + if (params->has_multifd_qatzip_level) { > + s->parameters.multifd_qatzip_level = params- > >multifd_qatzip_level; > + } > + if (params->has_multifd_qatzip_sw_fallback) { > + s->parameters.multifd_qatzip_sw_fallback = > + params->multifd_qatzip_sw_fallback; > + } > if (params->has_xbzrle_cache_size) { > s->parameters.xbzrle_cache_size = params->xbzrle_cache_size; > xbzrle_cache_resize(params->xbzrle_cache_size, errp); > diff --git a/migration/options.h b/migration/options.h > index 246c160aee..94aee24d97 100644 > --- a/migration/options.h > +++ b/migration/options.h > @@ -87,6 +87,8 @@ MigMode migrate_mode(void); > int migrate_multifd_channels(void); > MultiFDCompression migrate_multifd_compression(void); > int migrate_multifd_zlib_level(void); > +int migrate_multifd_qatzip_level(void); > +bool migrate_multifd_qatzip_sw_fallback(void); > int migrate_multifd_zstd_level(void); > uint8_t migrate_throttle_trigger_threshold(void); > const char *migrate_tls_authz(void); > diff --git a/qapi/migration.json b/qapi/migration.json > index 0b33a71ab4..66ea6d32fc 100644 > --- a/qapi/migration.json > +++ b/qapi/migration.json > @@ -853,6 +853,16 @@ > # speed, and 9 means best compression ratio which will consume > # more CPU. Defaults to 1. (Since 5.0) > # > +# @multifd-qatzip-level: Set the compression level to be used in live > +# migration. The level is an integer between 1 and 9, where 1 means > +# the best compression speed, and 9 means the best compression > +# ratio which will consume more CPU. Defaults to 1. > +# > +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware > +# is unavailable. Defaults to false. Software fallback performance > +# is very poor compared to regular zlib, so be cautious about > +# enabling this option. > +# > # @multifd-zstd-level: Set the compression level to be used in live > # migration, the compression level is an integer between 0 and 20, > # where 0 means no compression, 1 means the best compression > @@ -915,6 +925,7 @@ > 'xbzrle-cache-size', 'max-postcopy-bandwidth', > 'max-cpu-throttle', 'multifd-compression', > 'multifd-zlib-level', 'multifd-zstd-level', > + 'multifd-qatzip-level', 'multifd-qatzip-sw-fallback', > 'block-bitmap-mapping', > { 'name': 'x-vcpu-dirty-limit-period', 'features': > ['unstable'] }, > 'vcpu-dirty-limit', > @@ -1045,6 +1056,16 @@ > # speed, and 9 means best compression ratio which will consume > # more CPU. Defaults to 1. (Since 5.0) > # > +# @multifd-qatzip-level: Set the compression level to be used in live > +# migration. The level is an integer between 1 and 9, where 1 means > +# the best compression speed, and 9 means the best compression > +# ratio which will consume more CPU. Defaults to 1. > +# > +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware > +# is unavailable. Defaults to false. Software fallback performance > +# is very poor compared to regular zlib, so be cautious about > +# enabling this option. > +# > # @multifd-zstd-level: Set the compression level to be used in live > # migration, the compression level is an integer between 0 and 20, > # where 0 means no compression, 1 means the best compression > @@ -1125,6 +1146,8 @@ > '*max-cpu-throttle': 'uint8', > '*multifd-compression': 'MultiFDCompression', > '*multifd-zlib-level': 'uint8', > + '*multifd-qatzip-level': 'uint8', > + '*multifd-qatzip-sw-fallback': 'bool', > '*multifd-zstd-level': 'uint8', > '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], > '*x-vcpu-dirty-limit-period': { 'type': 'uint64', > @@ -1273,6 +1296,16 @@ > # speed, and 9 means best compression ratio which will consume > # more CPU. Defaults to 1. (Since 5.0) > # > +# @multifd-qatzip-level: Set the compression level to be used in live > +# migration. The level is an integer between 1 and 9, where 1 means > +# the best compression speed, and 9 means the best compression > +# ratio which will consume more CPU. Defaults to 1. > +# > +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware > +# is unavailable. Defaults to false. Software fallback performance > +# is very poor compared to regular zlib, so be cautious about > +# enabling this option. > +# > # @multifd-zstd-level: Set the compression level to be used in live > # migration, the compression level is an integer between 0 and 20, > # where 0 means no compression, 1 means the best compression > @@ -1350,6 +1383,8 @@ > '*max-cpu-throttle': 'uint8', > '*multifd-compression': 'MultiFDCompression', > '*multifd-zlib-level': 'uint8', > + '*multifd-qatzip-level': 'uint8', > + '*multifd-qatzip-sw-fallback': 'bool', > '*multifd-zstd-level': 'uint8', > '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], > '*x-vcpu-dirty-limit-period': { 'type': 'uint64', > -- > 2.30.2
Bryan Zhang <bryan.zhang@bytedance.com> writes: > Adds support for migration parameters to control QATzip compression > level and to enable/disable software fallback when QAT hardware is > unavailable. This is a preparatory commit for a subsequent commit that > will actually use QATzip compression. > > Signed-off-by: Bryan Zhang <bryan.zhang@bytedance.com> > Signed-off-by: Hao Xiang <hao.xiang@linux.dev> > --- > Revision: This commit now includes a parameter for controlling software > fallback. Fallback is generally intended to be disabled, but having this > option available enables using software fallback for testing. > > This commit also now has some glue code to properly set parameters. > > migration/migration-hmp-cmds.c | 8 +++++ > migration/options.c | 57 ++++++++++++++++++++++++++++++++++ > migration/options.h | 2 ++ > qapi/migration.json | 35 +++++++++++++++++++++ > 4 files changed, 102 insertions(+) > > diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c > index 99b49df5dd..4bd23ba14d 100644 > --- a/migration/migration-hmp-cmds.c > +++ b/migration/migration-hmp-cmds.c > @@ -630,6 +630,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict) > p->has_multifd_zlib_level = true; > visit_type_uint8(v, param, &p->multifd_zlib_level, &err); > break; > + case MIGRATION_PARAMETER_MULTIFD_QATZIP_LEVEL: > + p->has_multifd_qatzip_level = true; > + visit_type_uint8(v, param, &p->multifd_qatzip_level, &err); > + break; > + case MIGRATION_PARAMETER_MULTIFD_QATZIP_SW_FALLBACK: > + p->has_multifd_qatzip_sw_fallback = true; > + visit_type_bool(v, param, &p->multifd_qatzip_sw_fallback, &err); > + break; > case MIGRATION_PARAMETER_MULTIFD_ZSTD_LEVEL: > p->has_multifd_zstd_level = true; > visit_type_uint8(v, param, &p->multifd_zstd_level, &err); > diff --git a/migration/options.c b/migration/options.c > index 3e3e0b93b4..1316ea605a 100644 > --- a/migration/options.c > +++ b/migration/options.c > @@ -62,6 +62,15 @@ > #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE > /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */ > #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1 > +/* > + * 1: best speed, ... 9: best compress ratio > + * There is some nuance here. Refer to QATzip documentation to understand > + * the mapping of QATzip levels to standard deflate levels. > + */ > +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL 1 > +/* QATzip's SW fallback implementation is extremely slow, so avoid fallback */ > +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK false > + > /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */ > #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1 > > @@ -143,6 +152,12 @@ Property migration_properties[] = { > DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState, > parameters.multifd_zlib_level, > DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL), > + DEFINE_PROP_UINT8("multifd-qatzip-level", MigrationState, > + parameters.multifd_qatzip_level, > + DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL), > + DEFINE_PROP_BOOL("multifd-qatzip-sw-fallback", MigrationState, > + parameters.multifd_qatzip_sw_fallback, > + DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK), > DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState, > parameters.multifd_zstd_level, > DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL), > @@ -861,6 +876,20 @@ int migrate_multifd_zlib_level(void) > return s->parameters.multifd_zlib_level; > } > > +int migrate_multifd_qatzip_level(void) > +{ > + MigrationState *s = migrate_get_current(); > + > + return s->parameters.multifd_qatzip_level; > +} > + > +bool migrate_multifd_qatzip_sw_fallback(void) > +{ > + MigrationState *s = migrate_get_current(); > + > + return s->parameters.multifd_qatzip_sw_fallback; > +} > + > int migrate_multifd_zstd_level(void) > { > MigrationState *s = migrate_get_current(); > @@ -983,6 +1012,11 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) > params->multifd_compression = s->parameters.multifd_compression; > params->has_multifd_zlib_level = true; > params->multifd_zlib_level = s->parameters.multifd_zlib_level; > + params->has_multifd_qatzip_level = true; > + params->multifd_qatzip_level = s->parameters.multifd_qatzip_level; > + params->has_multifd_qatzip_sw_fallback = true; > + params->multifd_qatzip_sw_fallback = > + s->parameters.multifd_qatzip_sw_fallback; > params->has_multifd_zstd_level = true; > params->multifd_zstd_level = s->parameters.multifd_zstd_level; > params->has_xbzrle_cache_size = true; > @@ -1038,6 +1072,8 @@ void migrate_params_init(MigrationParameters *params) > params->has_multifd_channels = true; > params->has_multifd_compression = true; > params->has_multifd_zlib_level = true; > + params->has_multifd_qatzip_level = true; > + params->has_multifd_qatzip_sw_fallback = true; > params->has_multifd_zstd_level = true; > params->has_xbzrle_cache_size = true; > params->has_max_postcopy_bandwidth = true; > @@ -1147,6 +1183,14 @@ bool migrate_params_check(MigrationParameters *params, Error **errp) > return false; > } > > + if (params->has_multifd_qatzip_level && > + ((params->multifd_qatzip_level > 9) || > + (params->multifd_qatzip_level < 1))) { > + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_qatzip_level", > + "a value between 1 and 9"); > + return false; > + } > + > if (params->has_multifd_zstd_level && > (params->multifd_zstd_level > 20)) { > error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level", > @@ -1312,6 +1356,12 @@ static void migrate_params_test_apply(MigrateSetParameters *params, > if (params->has_multifd_compression) { > dest->multifd_compression = params->multifd_compression; > } > + if (params->has_multifd_qatzip_level) { > + dest->multifd_qatzip_level = params->multifd_qatzip_level; > + } > + if (params->has_multifd_qatzip_sw_fallback) { > + dest->multifd_qatzip_sw_fallback = params->multifd_qatzip_sw_fallback; > + } > if (params->has_xbzrle_cache_size) { > dest->xbzrle_cache_size = params->xbzrle_cache_size; > } > @@ -1447,6 +1497,13 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) > if (params->has_multifd_compression) { > s->parameters.multifd_compression = params->multifd_compression; > } > + if (params->has_multifd_qatzip_level) { > + s->parameters.multifd_qatzip_level = params->multifd_qatzip_level; > + } > + if (params->has_multifd_qatzip_sw_fallback) { > + s->parameters.multifd_qatzip_sw_fallback = > + params->multifd_qatzip_sw_fallback; > + } > if (params->has_xbzrle_cache_size) { > s->parameters.xbzrle_cache_size = params->xbzrle_cache_size; > xbzrle_cache_resize(params->xbzrle_cache_size, errp); > diff --git a/migration/options.h b/migration/options.h > index 246c160aee..94aee24d97 100644 > --- a/migration/options.h > +++ b/migration/options.h > @@ -87,6 +87,8 @@ MigMode migrate_mode(void); > int migrate_multifd_channels(void); > MultiFDCompression migrate_multifd_compression(void); > int migrate_multifd_zlib_level(void); > +int migrate_multifd_qatzip_level(void); > +bool migrate_multifd_qatzip_sw_fallback(void); > int migrate_multifd_zstd_level(void); > uint8_t migrate_throttle_trigger_threshold(void); > const char *migrate_tls_authz(void); > diff --git a/qapi/migration.json b/qapi/migration.json > index 0b33a71ab4..66ea6d32fc 100644 > --- a/qapi/migration.json > +++ b/qapi/migration.json > @@ -853,6 +853,16 @@ > # speed, and 9 means best compression ratio which will consume > # more CPU. Defaults to 1. (Since 5.0) > # > +# @multifd-qatzip-level: Set the compression level to be used in live > +# migration. The level is an integer between 1 and 9, where 1 means > +# the best compression speed, and 9 means the best compression > +# ratio which will consume more CPU. Defaults to 1. > +# > +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware > +# is unavailable. Defaults to false. Software fallback performance > +# is very poor compared to regular zlib, so be cautious about > +# enabling this option. > +# > # @multifd-zstd-level: Set the compression level to be used in live > # migration, the compression level is an integer between 0 and 20, > # where 0 means no compression, 1 means the best compression > @@ -915,6 +925,7 @@ > 'xbzrle-cache-size', 'max-postcopy-bandwidth', > 'max-cpu-throttle', 'multifd-compression', > 'multifd-zlib-level', 'multifd-zstd-level', > + 'multifd-qatzip-level', 'multifd-qatzip-sw-fallback', > 'block-bitmap-mapping', > { 'name': 'x-vcpu-dirty-limit-period', 'features': ['unstable'] }, > 'vcpu-dirty-limit', > @@ -1045,6 +1056,16 @@ > # speed, and 9 means best compression ratio which will consume > # more CPU. Defaults to 1. (Since 5.0) > # > +# @multifd-qatzip-level: Set the compression level to be used in live > +# migration. The level is an integer between 1 and 9, where 1 means > +# the best compression speed, and 9 means the best compression > +# ratio which will consume more CPU. Defaults to 1. (Since 9.1) > +# > +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware > +# is unavailable. Defaults to false. Software fallback performance > +# is very poor compared to regular zlib, so be cautious about > +# enabling this option. (Since 9.1) > +# > # @multifd-zstd-level: Set the compression level to be used in live > # migration, the compression level is an integer between 0 and 20, > # where 0 means no compression, 1 means the best compression > @@ -1125,6 +1146,8 @@ > '*max-cpu-throttle': 'uint8', > '*multifd-compression': 'MultiFDCompression', > '*multifd-zlib-level': 'uint8', > + '*multifd-qatzip-level': 'uint8', > + '*multifd-qatzip-sw-fallback': 'bool', > '*multifd-zstd-level': 'uint8', > '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], > '*x-vcpu-dirty-limit-period': { 'type': 'uint64', > @@ -1273,6 +1296,16 @@ > # speed, and 9 means best compression ratio which will consume > # more CPU. Defaults to 1. (Since 5.0) > # > +# @multifd-qatzip-level: Set the compression level to be used in live > +# migration. The level is an integer between 1 and 9, where 1 means > +# the best compression speed, and 9 means the best compression > +# ratio which will consume more CPU. Defaults to 1. (Since 9.1) > +# > +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware > +# is unavailable. Defaults to false. Software fallback performance > +# is very poor compared to regular zlib, so be cautious about > +# enabling this option. (Since 9.1) > +# > # @multifd-zstd-level: Set the compression level to be used in live > # migration, the compression level is an integer between 0 and 20, > # where 0 means no compression, 1 means the best compression > @@ -1350,6 +1383,8 @@ > '*max-cpu-throttle': 'uint8', > '*multifd-compression': 'MultiFDCompression', > '*multifd-zlib-level': 'uint8', > + '*multifd-qatzip-level': 'uint8', > + '*multifd-qatzip-sw-fallback': 'bool', > '*multifd-zstd-level': 'uint8', > '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], > '*x-vcpu-dirty-limit-period': { 'type': 'uint64',
> On Mar 28, 2024, at 12:23 AM, Liu, Yuan1 <yuan1.liu@intel.com> wrote: > >> -----Original Message----- >> From: Bryan Zhang <bryan.zhang@bytedance.com> >> Sent: Wednesday, March 27, 2024 6:42 AM >> To: qemu-devel@nongnu.org >> Cc: peterx@redhat.com; farosas@suse.de; Liu, Yuan1 <yuan1.liu@intel.com>; >> berrange@redhat.com; Zou, Nanhai <nanhai.zou@intel.com>; >> hao.xiang@linux.dev; Bryan Zhang <bryan.zhang@bytedance.com> >> Subject: [PATCH v2 2/5] migration: Add migration parameters for QATzip >> >> Adds support for migration parameters to control QATzip compression >> level and to enable/disable software fallback when QAT hardware is >> unavailable. This is a preparatory commit for a subsequent commit that >> will actually use QATzip compression. >> >> Signed-off-by: Bryan Zhang <bryan.zhang@bytedance.com> >> Signed-off-by: Hao Xiang <hao.xiang@linux.dev> >> --- >> Revision: This commit now includes a parameter for controlling software >> fallback. Fallback is generally intended to be disabled, but having this >> option available enables using software fallback for testing. >> >> This commit also now has some glue code to properly set parameters. >> >> migration/migration-hmp-cmds.c | 8 +++++ >> migration/options.c | 57 ++++++++++++++++++++++++++++++++++ >> migration/options.h | 2 ++ >> qapi/migration.json | 35 +++++++++++++++++++++ >> 4 files changed, 102 insertions(+) >> >> diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp- >> cmds.c >> index 99b49df5dd..4bd23ba14d 100644 >> --- a/migration/migration-hmp-cmds.c >> +++ b/migration/migration-hmp-cmds.c >> @@ -630,6 +630,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const >> QDict *qdict) >> p->has_multifd_zlib_level = true; >> visit_type_uint8(v, param, &p->multifd_zlib_level, &err); >> break; >> + case MIGRATION_PARAMETER_MULTIFD_QATZIP_LEVEL: >> + p->has_multifd_qatzip_level = true; >> + visit_type_uint8(v, param, &p->multifd_qatzip_level, &err); >> + break; >> + case MIGRATION_PARAMETER_MULTIFD_QATZIP_SW_FALLBACK: >> + p->has_multifd_qatzip_sw_fallback = true; >> + visit_type_bool(v, param, &p->multifd_qatzip_sw_fallback, &err); >> + break; >> case MIGRATION_PARAMETER_MULTIFD_ZSTD_LEVEL: >> p->has_multifd_zstd_level = true; >> visit_type_uint8(v, param, &p->multifd_zstd_level, &err); >> diff --git a/migration/options.c b/migration/options.c >> index 3e3e0b93b4..1316ea605a 100644 >> --- a/migration/options.c >> +++ b/migration/options.c >> @@ -62,6 +62,15 @@ >> #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE >> /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */ >> #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1 >> +/* >> + * 1: best speed, ... 9: best compress ratio >> + * There is some nuance here. Refer to QATzip documentation to understand >> + * the mapping of QATzip levels to standard deflate levels. >> + */ >> +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL 1 >> +/* QATzip's SW fallback implementation is extremely slow, so avoid >> fallback */ >> +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK false >> + >> /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */ >> #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1 > > Hi Bryan > > The default compression level may be set higher, such as 6. I checked QAT throughput > If the data size is less than or equal to 64K, level 1 has much better throughput > performance than level 6 and level 9. But if the data size is greater than 128K, little > change in throughput, and the default MULTIFD_PACKET_SIZE is 512K, you can have a try > to use a high compression level, to get better compression performance without affecting > throughput. > > In addition, if you change MULTIFD_PACKET_SIZE to 64K, you may have better throughput > with more multifd threads Hi Yuan, Thanks for your comment. I did a quick experiments on our CPU. With chunk size at 512K, for silesia dataset, the throughput degrades at level 6: # for f in `seq 1 1 9`; do echo "Level $f:"; qzip -C 524288 -L $f -k silesia.8G | grep ":"; done Level 1: Time taken: 2230.772 ms Throughput: 30402.186 Mbit/s Space Savings: 65.712 % Compression ratio: 2.916 : 1 Level 2: Time taken: 2245.205 ms Throughput: 30206.750 Mbit/s Space Savings: 65.712 % Compression ratio: 2.916 : 1 Level 3: Time taken: 2217.789 ms Throughput: 30580.161 Mbit/s Space Savings: 65.712 % Compression ratio: 2.916 : 1 Level 4: Time taken: 2251.014 ms Throughput: 30128.798 Mbit/s Space Savings: 65.712 % Compression ratio: 2.916 : 1 Level 5: Time taken: 2200.991 ms Throughput: 30813.550 Mbit/s Space Savings: 65.712 % Compression ratio: 2.916 : 1 Level 6: Time taken: 2508.218 ms Throughput: 27039.255 Mbit/s Space Savings: 67.396 % Compression ratio: 3.067 : 1 Level 7: Time taken: 2510.847 ms Throughput: 27010.943 Mbit/s Space Savings: 67.396 % Compression ratio: 3.067 : 1 Level 8: Time taken: 2521.428 ms Throughput: 26897.594 Mbit/s Space Savings: 67.396 % Compression ratio: 3.067 : 1 Level 9: Time taken: 3071.055 ms Throughput: 22083.729 Mbit/s Space Savings: 67.664 % Compression ratio: 3.092 : 1 For random text data, throughput actually improves at level 6. # for f in `seq 1 1 9`; do echo "Level $f:"; qzip -C 524288 -L $f -k text.txt | grep ":"; done Level 1: Time taken: 1788.683 ms Throughput: 8945.129 Mbit/s Space Savings: 24.959 % Compression ratio: 1.333 : 1 Level 2: Time taken: 1786.135 ms Throughput: 8957.890 Mbit/s Space Savings: 24.959 % Compression ratio: 1.333 : 1 Level 3: Time taken: 1785.564 ms Throughput: 8960.754 Mbit/s Space Savings: 24.959 % Compression ratio: 1.333 : 1 Level 4: Time taken: 1787.351 ms Throughput: 8951.795 Mbit/s Space Savings: 24.959 % Compression ratio: 1.333 : 1 Level 5: Time taken: 1785.171 ms Throughput: 8962.727 Mbit/s Space Savings: 24.959 % Compression ratio: 1.333 : 1 Level 6: Time taken: 1752.000 ms Throughput: 9132.420 Mbit/s Space Savings: 24.873 % Compression ratio: 1.331 : 1 Level 7: Time taken: 1752.297 ms Throughput: 9130.872 Mbit/s Space Savings: 24.873 % Compression ratio: 1.331 : 1 Level 8: Time taken: 1752.538 ms Throughput: 9129.617 Mbit/s Space Savings: 24.873 % Compression ratio: 1.331 : 1 Level 9: Time taken: 1762.593 ms Throughput: 9077.535 Mbit/s Space Savings: 24.870 % Compression ratio: 1.331 : 1 For random binary data, throughput remains the same from 0-9. Given our live migration is mostly memory pages, and I am not very sure what scenario would better fit to above tests. I am OK with either default level (1 or 6). I can change to 6 in my next patchiest if you believe that is better. Thanks very much! Regards, Yichen > >> @@ -143,6 +152,12 @@ Property migration_properties[] = { >> DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState, >> parameters.multifd_zlib_level, >> DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL), >> + DEFINE_PROP_UINT8("multifd-qatzip-level", MigrationState, >> + parameters.multifd_qatzip_level, >> + DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL), >> + DEFINE_PROP_BOOL("multifd-qatzip-sw-fallback", MigrationState, >> + parameters.multifd_qatzip_sw_fallback, >> + DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK), >> DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState, >> parameters.multifd_zstd_level, >> DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL), >> @@ -861,6 +876,20 @@ int migrate_multifd_zlib_level(void) >> return s->parameters.multifd_zlib_level; >> } >> >> +int migrate_multifd_qatzip_level(void) >> +{ >> + MigrationState *s = migrate_get_current(); >> + >> + return s->parameters.multifd_qatzip_level; >> +} >> + >> +bool migrate_multifd_qatzip_sw_fallback(void) >> +{ >> + MigrationState *s = migrate_get_current(); >> + >> + return s->parameters.multifd_qatzip_sw_fallback; >> +} >> + >> int migrate_multifd_zstd_level(void) >> { >> MigrationState *s = migrate_get_current(); >> @@ -983,6 +1012,11 @@ MigrationParameters >> *qmp_query_migrate_parameters(Error **errp) >> params->multifd_compression = s->parameters.multifd_compression; >> params->has_multifd_zlib_level = true; >> params->multifd_zlib_level = s->parameters.multifd_zlib_level; >> + params->has_multifd_qatzip_level = true; >> + params->multifd_qatzip_level = s->parameters.multifd_qatzip_level; >> + params->has_multifd_qatzip_sw_fallback = true; >> + params->multifd_qatzip_sw_fallback = >> + s->parameters.multifd_qatzip_sw_fallback; >> params->has_multifd_zstd_level = true; >> params->multifd_zstd_level = s->parameters.multifd_zstd_level; >> params->has_xbzrle_cache_size = true; >> @@ -1038,6 +1072,8 @@ void migrate_params_init(MigrationParameters >> *params) >> params->has_multifd_channels = true; >> params->has_multifd_compression = true; >> params->has_multifd_zlib_level = true; >> + params->has_multifd_qatzip_level = true; >> + params->has_multifd_qatzip_sw_fallback = true; >> params->has_multifd_zstd_level = true; >> params->has_xbzrle_cache_size = true; >> params->has_max_postcopy_bandwidth = true; >> @@ -1147,6 +1183,14 @@ bool migrate_params_check(MigrationParameters >> *params, Error **errp) >> return false; >> } >> >> + if (params->has_multifd_qatzip_level && >> + ((params->multifd_qatzip_level > 9) || >> + (params->multifd_qatzip_level < 1))) { >> + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, >> "multifd_qatzip_level", >> + "a value between 1 and 9"); >> + return false; >> + } >> + >> if (params->has_multifd_zstd_level && >> (params->multifd_zstd_level > 20)) { >> error_setg(errp, QERR_INVALID_PARAMETER_VALUE, >> "multifd_zstd_level", >> @@ -1312,6 +1356,12 @@ static void >> migrate_params_test_apply(MigrateSetParameters *params, >> if (params->has_multifd_compression) { >> dest->multifd_compression = params->multifd_compression; >> } >> + if (params->has_multifd_qatzip_level) { >> + dest->multifd_qatzip_level = params->multifd_qatzip_level; >> + } >> + if (params->has_multifd_qatzip_sw_fallback) { >> + dest->multifd_qatzip_sw_fallback = params- >>> multifd_qatzip_sw_fallback; >> + } >> if (params->has_xbzrle_cache_size) { >> dest->xbzrle_cache_size = params->xbzrle_cache_size; >> } >> @@ -1447,6 +1497,13 @@ static void >> migrate_params_apply(MigrateSetParameters *params, Error **errp) >> if (params->has_multifd_compression) { >> s->parameters.multifd_compression = params->multifd_compression; >> } >> + if (params->has_multifd_qatzip_level) { >> + s->parameters.multifd_qatzip_level = params- >>> multifd_qatzip_level; >> + } >> + if (params->has_multifd_qatzip_sw_fallback) { >> + s->parameters.multifd_qatzip_sw_fallback = >> + params->multifd_qatzip_sw_fallback; >> + } >> if (params->has_xbzrle_cache_size) { >> s->parameters.xbzrle_cache_size = params->xbzrle_cache_size; >> xbzrle_cache_resize(params->xbzrle_cache_size, errp); >> diff --git a/migration/options.h b/migration/options.h >> index 246c160aee..94aee24d97 100644 >> --- a/migration/options.h >> +++ b/migration/options.h >> @@ -87,6 +87,8 @@ MigMode migrate_mode(void); >> int migrate_multifd_channels(void); >> MultiFDCompression migrate_multifd_compression(void); >> int migrate_multifd_zlib_level(void); >> +int migrate_multifd_qatzip_level(void); >> +bool migrate_multifd_qatzip_sw_fallback(void); >> int migrate_multifd_zstd_level(void); >> uint8_t migrate_throttle_trigger_threshold(void); >> const char *migrate_tls_authz(void); >> diff --git a/qapi/migration.json b/qapi/migration.json >> index 0b33a71ab4..66ea6d32fc 100644 >> --- a/qapi/migration.json >> +++ b/qapi/migration.json >> @@ -853,6 +853,16 @@ >> # speed, and 9 means best compression ratio which will consume >> # more CPU. Defaults to 1. (Since 5.0) >> # >> +# @multifd-qatzip-level: Set the compression level to be used in live >> +# migration. The level is an integer between 1 and 9, where 1 means >> +# the best compression speed, and 9 means the best compression >> +# ratio which will consume more CPU. Defaults to 1. >> +# >> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware >> +# is unavailable. Defaults to false. Software fallback performance >> +# is very poor compared to regular zlib, so be cautious about >> +# enabling this option. >> +# >> # @multifd-zstd-level: Set the compression level to be used in live >> # migration, the compression level is an integer between 0 and 20, >> # where 0 means no compression, 1 means the best compression >> @@ -915,6 +925,7 @@ >> 'xbzrle-cache-size', 'max-postcopy-bandwidth', >> 'max-cpu-throttle', 'multifd-compression', >> 'multifd-zlib-level', 'multifd-zstd-level', >> + 'multifd-qatzip-level', 'multifd-qatzip-sw-fallback', >> 'block-bitmap-mapping', >> { 'name': 'x-vcpu-dirty-limit-period', 'features': >> ['unstable'] }, >> 'vcpu-dirty-limit', >> @@ -1045,6 +1056,16 @@ >> # speed, and 9 means best compression ratio which will consume >> # more CPU. Defaults to 1. (Since 5.0) >> # >> +# @multifd-qatzip-level: Set the compression level to be used in live >> +# migration. The level is an integer between 1 and 9, where 1 means >> +# the best compression speed, and 9 means the best compression >> +# ratio which will consume more CPU. Defaults to 1. >> +# >> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware >> +# is unavailable. Defaults to false. Software fallback performance >> +# is very poor compared to regular zlib, so be cautious about >> +# enabling this option. >> +# >> # @multifd-zstd-level: Set the compression level to be used in live >> # migration, the compression level is an integer between 0 and 20, >> # where 0 means no compression, 1 means the best compression >> @@ -1125,6 +1146,8 @@ >> '*max-cpu-throttle': 'uint8', >> '*multifd-compression': 'MultiFDCompression', >> '*multifd-zlib-level': 'uint8', >> + '*multifd-qatzip-level': 'uint8', >> + '*multifd-qatzip-sw-fallback': 'bool', >> '*multifd-zstd-level': 'uint8', >> '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], >> '*x-vcpu-dirty-limit-period': { 'type': 'uint64', >> @@ -1273,6 +1296,16 @@ >> # speed, and 9 means best compression ratio which will consume >> # more CPU. Defaults to 1. (Since 5.0) >> # >> +# @multifd-qatzip-level: Set the compression level to be used in live >> +# migration. The level is an integer between 1 and 9, where 1 means >> +# the best compression speed, and 9 means the best compression >> +# ratio which will consume more CPU. Defaults to 1. >> +# >> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware >> +# is unavailable. Defaults to false. Software fallback performance >> +# is very poor compared to regular zlib, so be cautious about >> +# enabling this option. >> +# >> # @multifd-zstd-level: Set the compression level to be used in live >> # migration, the compression level is an integer between 0 and 20, >> # where 0 means no compression, 1 means the best compression >> @@ -1350,6 +1383,8 @@ >> '*max-cpu-throttle': 'uint8', >> '*multifd-compression': 'MultiFDCompression', >> '*multifd-zlib-level': 'uint8', >> + '*multifd-qatzip-level': 'uint8', >> + '*multifd-qatzip-sw-fallback': 'bool', >> '*multifd-zstd-level': 'uint8', >> '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], >> '*x-vcpu-dirty-limit-period': { 'type': 'uint64', >> -- >> 2.30.2
> -----Original Message----- > From: Yichen Wang <yichen.wang@bytedance.com> > Sent: Thursday, June 27, 2024 8:17 AM > To: Liu, Yuan1 <yuan1.liu@intel.com> > Cc: Bryan Zhang <bryan.zhang@bytedance.com>; qemu-devel@nongnu.org; > peterx@redhat.com; farosas@suse.de; berrange@redhat.com; Zou, Nanhai > <nanhai.zou@intel.com>; hao.xiang@linux.dev > Subject: Re: [PATCH v2 2/5] migration: Add migration parameters for QATzip > > > > > On Mar 28, 2024, at 12:23 AM, Liu, Yuan1 <yuan1.liu@intel.com> wrote: > > > >> -----Original Message----- > >> From: Bryan Zhang <bryan.zhang@bytedance.com> > >> Sent: Wednesday, March 27, 2024 6:42 AM > >> To: qemu-devel@nongnu.org > >> Cc: peterx@redhat.com; farosas@suse.de; Liu, Yuan1 > <yuan1.liu@intel.com>; > >> berrange@redhat.com; Zou, Nanhai <nanhai.zou@intel.com>; > >> hao.xiang@linux.dev; Bryan Zhang <bryan.zhang@bytedance.com> > >> Subject: [PATCH v2 2/5] migration: Add migration parameters for QATzip > >> > >> Adds support for migration parameters to control QATzip compression > >> level and to enable/disable software fallback when QAT hardware is > >> unavailable. This is a preparatory commit for a subsequent commit that > >> will actually use QATzip compression. > >> > >> Signed-off-by: Bryan Zhang <bryan.zhang@bytedance.com> > >> Signed-off-by: Hao Xiang <hao.xiang@linux.dev> > >> --- > >> Revision: This commit now includes a parameter for controlling software > >> fallback. Fallback is generally intended to be disabled, but having > this > >> option available enables using software fallback for testing. > >> > >> This commit also now has some glue code to properly set parameters. > >> > >> migration/migration-hmp-cmds.c | 8 +++++ > >> migration/options.c | 57 ++++++++++++++++++++++++++++++++++ > >> migration/options.h | 2 ++ > >> qapi/migration.json | 35 +++++++++++++++++++++ > >> 4 files changed, 102 insertions(+) > >> > >> diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp- > >> cmds.c > >> index 99b49df5dd..4bd23ba14d 100644 > >> --- a/migration/migration-hmp-cmds.c > >> +++ b/migration/migration-hmp-cmds.c > >> @@ -630,6 +630,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const > >> QDict *qdict) > >> p->has_multifd_zlib_level = true; > >> visit_type_uint8(v, param, &p->multifd_zlib_level, &err); > >> break; > >> + case MIGRATION_PARAMETER_MULTIFD_QATZIP_LEVEL: > >> + p->has_multifd_qatzip_level = true; > >> + visit_type_uint8(v, param, &p->multifd_qatzip_level, &err); > >> + break; > >> + case MIGRATION_PARAMETER_MULTIFD_QATZIP_SW_FALLBACK: > >> + p->has_multifd_qatzip_sw_fallback = true; > >> + visit_type_bool(v, param, &p->multifd_qatzip_sw_fallback, > &err); > >> + break; > >> case MIGRATION_PARAMETER_MULTIFD_ZSTD_LEVEL: > >> p->has_multifd_zstd_level = true; > >> visit_type_uint8(v, param, &p->multifd_zstd_level, &err); > >> diff --git a/migration/options.c b/migration/options.c > >> index 3e3e0b93b4..1316ea605a 100644 > >> --- a/migration/options.c > >> +++ b/migration/options.c > >> @@ -62,6 +62,15 @@ > >> #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE > >> /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */ > >> #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1 > >> +/* > >> + * 1: best speed, ... 9: best compress ratio > >> + * There is some nuance here. Refer to QATzip documentation to > understand > >> + * the mapping of QATzip levels to standard deflate levels. > >> + */ > >> +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL 1 > >> +/* QATzip's SW fallback implementation is extremely slow, so avoid > >> fallback */ > >> +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK false > >> + > >> /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */ > >> #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1 > > > > Hi Bryan > > > > The default compression level may be set higher, such as 6. I checked > QAT throughput > > If the data size is less than or equal to 64K, level 1 has much better > throughput > > performance than level 6 and level 9. But if the data size is greater > than 128K, little > > change in throughput, and the default MULTIFD_PACKET_SIZE is 512K, you > can have a try > > to use a high compression level, to get better compression performance > without affecting > > throughput. > > > > In addition, if you change MULTIFD_PACKET_SIZE to 64K, you may have > better throughput > > with more multifd threads > Hi Yuan, > > Thanks for your comment. I did a quick experiments on our CPU. With chunk > size at 512K, for silesia dataset, the throughput degrades at level 6: > # for f in `seq 1 1 9`; do echo "Level $f:"; qzip -C 524288 -L $f -k > silesia.8G | grep ":"; done > Level 1: > Time taken: 2230.772 ms > Throughput: 30402.186 Mbit/s > Space Savings: 65.712 % > Compression ratio: 2.916 : 1 > Level 2: > Time taken: 2245.205 ms > Throughput: 30206.750 Mbit/s > Space Savings: 65.712 % > Compression ratio: 2.916 : 1 > Level 3: > Time taken: 2217.789 ms > Throughput: 30580.161 Mbit/s > Space Savings: 65.712 % > Compression ratio: 2.916 : 1 > Level 4: > Time taken: 2251.014 ms > Throughput: 30128.798 Mbit/s > Space Savings: 65.712 % > Compression ratio: 2.916 : 1 > Level 5: > Time taken: 2200.991 ms > Throughput: 30813.550 Mbit/s > Space Savings: 65.712 % > Compression ratio: 2.916 : 1 > Level 6: > Time taken: 2508.218 ms > Throughput: 27039.255 Mbit/s > Space Savings: 67.396 % > Compression ratio: 3.067 : 1 > Level 7: > Time taken: 2510.847 ms > Throughput: 27010.943 Mbit/s > Space Savings: 67.396 % > Compression ratio: 3.067 : 1 > Level 8: > Time taken: 2521.428 ms > Throughput: 26897.594 Mbit/s > Space Savings: 67.396 % > Compression ratio: 3.067 : 1 > Level 9: > Time taken: 3071.055 ms > Throughput: 22083.729 Mbit/s > Space Savings: 67.664 % > Compression ratio: 3.092 : 1 > > For random text data, throughput actually improves at level 6. > # for f in `seq 1 1 9`; do echo "Level $f:"; qzip -C 524288 -L $f -k > text.txt | grep ":"; done > Level 1: > Time taken: 1788.683 ms > Throughput: 8945.129 Mbit/s > Space Savings: 24.959 % > Compression ratio: 1.333 : 1 > Level 2: > Time taken: 1786.135 ms > Throughput: 8957.890 Mbit/s > Space Savings: 24.959 % > Compression ratio: 1.333 : 1 > Level 3: > Time taken: 1785.564 ms > Throughput: 8960.754 Mbit/s > Space Savings: 24.959 % > Compression ratio: 1.333 : 1 > Level 4: > Time taken: 1787.351 ms > Throughput: 8951.795 Mbit/s > Space Savings: 24.959 % > Compression ratio: 1.333 : 1 > Level 5: > Time taken: 1785.171 ms > Throughput: 8962.727 Mbit/s > Space Savings: 24.959 % > Compression ratio: 1.333 : 1 > Level 6: > Time taken: 1752.000 ms > Throughput: 9132.420 Mbit/s > Space Savings: 24.873 % > Compression ratio: 1.331 : 1 > Level 7: > Time taken: 1752.297 ms > Throughput: 9130.872 Mbit/s > Space Savings: 24.873 % > Compression ratio: 1.331 : 1 > Level 8: > Time taken: 1752.538 ms > Throughput: 9129.617 Mbit/s > Space Savings: 24.873 % > Compression ratio: 1.331 : 1 > Level 9: > Time taken: 1762.593 ms > Throughput: 9077.535 Mbit/s > Space Savings: 24.870 % > Compression ratio: 1.331 : 1 > > For random binary data, throughput remains the same from 0-9. > > Given our live migration is mostly memory pages, and I am not very sure > what scenario would better fit to above tests. I am OK with either default > level (1 or 6). I can change to 6 in my next patchiest if you believe that > is better. Please use level 1 as the default compression level, my reason below 1. The level 6 does not significantly improve the compression ratio than level 1 for Silesia dataset But the throughput dropped by 10%. I also test Calgary Corpus dataset, same with Silesia. 2. Users can modify the actual compression level according to multifd_qatzip_level. > >> @@ -143,6 +152,12 @@ Property migration_properties[] = { > >> DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState, > >> parameters.multifd_zlib_level, > >> DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL), > >> + DEFINE_PROP_UINT8("multifd-qatzip-level", MigrationState, > >> + parameters.multifd_qatzip_level, > >> + DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL), > >> + DEFINE_PROP_BOOL("multifd-qatzip-sw-fallback", MigrationState, > >> + parameters.multifd_qatzip_sw_fallback, > >> + DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK), > >> DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState, > >> parameters.multifd_zstd_level, > >> DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL), > >> @@ -861,6 +876,20 @@ int migrate_multifd_zlib_level(void) > >> return s->parameters.multifd_zlib_level; > >> } > >> > >> +int migrate_multifd_qatzip_level(void) > >> +{ > >> + MigrationState *s = migrate_get_current(); > >> + > >> + return s->parameters.multifd_qatzip_level; > >> +} > >> + > >> +bool migrate_multifd_qatzip_sw_fallback(void) > >> +{ > >> + MigrationState *s = migrate_get_current(); > >> + > >> + return s->parameters.multifd_qatzip_sw_fallback; > >> +} > >> + > >> int migrate_multifd_zstd_level(void) > >> { > >> MigrationState *s = migrate_get_current(); > >> @@ -983,6 +1012,11 @@ MigrationParameters > >> *qmp_query_migrate_parameters(Error **errp) > >> params->multifd_compression = s->parameters.multifd_compression; > >> params->has_multifd_zlib_level = true; > >> params->multifd_zlib_level = s->parameters.multifd_zlib_level; > >> + params->has_multifd_qatzip_level = true; > >> + params->multifd_qatzip_level = s->parameters.multifd_qatzip_level; > >> + params->has_multifd_qatzip_sw_fallback = true; > >> + params->multifd_qatzip_sw_fallback = > >> + s->parameters.multifd_qatzip_sw_fallback; > >> params->has_multifd_zstd_level = true; > >> params->multifd_zstd_level = s->parameters.multifd_zstd_level; > >> params->has_xbzrle_cache_size = true; > >> @@ -1038,6 +1072,8 @@ void migrate_params_init(MigrationParameters > >> *params) > >> params->has_multifd_channels = true; > >> params->has_multifd_compression = true; > >> params->has_multifd_zlib_level = true; > >> + params->has_multifd_qatzip_level = true; > >> + params->has_multifd_qatzip_sw_fallback = true; > >> params->has_multifd_zstd_level = true; > >> params->has_xbzrle_cache_size = true; > >> params->has_max_postcopy_bandwidth = true; > >> @@ -1147,6 +1183,14 @@ bool migrate_params_check(MigrationParameters > >> *params, Error **errp) > >> return false; > >> } > >> > >> + if (params->has_multifd_qatzip_level && > >> + ((params->multifd_qatzip_level > 9) || > >> + (params->multifd_qatzip_level < 1))) { > >> + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, > >> "multifd_qatzip_level", > >> + "a value between 1 and 9"); > >> + return false; > >> + } > >> + > >> if (params->has_multifd_zstd_level && > >> (params->multifd_zstd_level > 20)) { > >> error_setg(errp, QERR_INVALID_PARAMETER_VALUE, > >> "multifd_zstd_level", > >> @@ -1312,6 +1356,12 @@ static void > >> migrate_params_test_apply(MigrateSetParameters *params, > >> if (params->has_multifd_compression) { > >> dest->multifd_compression = params->multifd_compression; > >> } > >> + if (params->has_multifd_qatzip_level) { > >> + dest->multifd_qatzip_level = params->multifd_qatzip_level; > >> + } > >> + if (params->has_multifd_qatzip_sw_fallback) { > >> + dest->multifd_qatzip_sw_fallback = params- > >>> multifd_qatzip_sw_fallback; > >> + } > >> if (params->has_xbzrle_cache_size) { > >> dest->xbzrle_cache_size = params->xbzrle_cache_size; > >> } > >> @@ -1447,6 +1497,13 @@ static void > >> migrate_params_apply(MigrateSetParameters *params, Error **errp) > >> if (params->has_multifd_compression) { > >> s->parameters.multifd_compression = params- > >multifd_compression; > >> } > >> + if (params->has_multifd_qatzip_level) { > >> + s->parameters.multifd_qatzip_level = params- > >>> multifd_qatzip_level; > >> + } > >> + if (params->has_multifd_qatzip_sw_fallback) { > >> + s->parameters.multifd_qatzip_sw_fallback = > >> + params->multifd_qatzip_sw_fallback; > >> + } > >> if (params->has_xbzrle_cache_size) { > >> s->parameters.xbzrle_cache_size = params->xbzrle_cache_size; > >> xbzrle_cache_resize(params->xbzrle_cache_size, errp); > >> diff --git a/migration/options.h b/migration/options.h > >> index 246c160aee..94aee24d97 100644 > >> --- a/migration/options.h > >> +++ b/migration/options.h > >> @@ -87,6 +87,8 @@ MigMode migrate_mode(void); > >> int migrate_multifd_channels(void); > >> MultiFDCompression migrate_multifd_compression(void); > >> int migrate_multifd_zlib_level(void); > >> +int migrate_multifd_qatzip_level(void); > >> +bool migrate_multifd_qatzip_sw_fallback(void); > >> int migrate_multifd_zstd_level(void); > >> uint8_t migrate_throttle_trigger_threshold(void); > >> const char *migrate_tls_authz(void); > >> diff --git a/qapi/migration.json b/qapi/migration.json > >> index 0b33a71ab4..66ea6d32fc 100644 > >> --- a/qapi/migration.json > >> +++ b/qapi/migration.json > >> @@ -853,6 +853,16 @@ > >> # speed, and 9 means best compression ratio which will consume > >> # more CPU. Defaults to 1. (Since 5.0) > >> # > >> +# @multifd-qatzip-level: Set the compression level to be used in live > >> +# migration. The level is an integer between 1 and 9, where 1 > means > >> +# the best compression speed, and 9 means the best compression > >> +# ratio which will consume more CPU. Defaults to 1. > >> +# > >> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT > hardware > >> +# is unavailable. Defaults to false. Software fallback performance > >> +# is very poor compared to regular zlib, so be cautious about > >> +# enabling this option. > >> +# > >> # @multifd-zstd-level: Set the compression level to be used in live > >> # migration, the compression level is an integer between 0 and 20, > >> # where 0 means no compression, 1 means the best compression > >> @@ -915,6 +925,7 @@ > >> 'xbzrle-cache-size', 'max-postcopy-bandwidth', > >> 'max-cpu-throttle', 'multifd-compression', > >> 'multifd-zlib-level', 'multifd-zstd-level', > >> + 'multifd-qatzip-level', 'multifd-qatzip-sw-fallback', > >> 'block-bitmap-mapping', > >> { 'name': 'x-vcpu-dirty-limit-period', 'features': > >> ['unstable'] }, > >> 'vcpu-dirty-limit', > >> @@ -1045,6 +1056,16 @@ > >> # speed, and 9 means best compression ratio which will consume > >> # more CPU. Defaults to 1. (Since 5.0) > >> # > >> +# @multifd-qatzip-level: Set the compression level to be used in live > >> +# migration. The level is an integer between 1 and 9, where 1 > means > >> +# the best compression speed, and 9 means the best compression > >> +# ratio which will consume more CPU. Defaults to 1. > >> +# > >> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT > hardware > >> +# is unavailable. Defaults to false. Software fallback performance > >> +# is very poor compared to regular zlib, so be cautious about > >> +# enabling this option. > >> +# > >> # @multifd-zstd-level: Set the compression level to be used in live > >> # migration, the compression level is an integer between 0 and 20, > >> # where 0 means no compression, 1 means the best compression > >> @@ -1125,6 +1146,8 @@ > >> '*max-cpu-throttle': 'uint8', > >> '*multifd-compression': 'MultiFDCompression', > >> '*multifd-zlib-level': 'uint8', > >> + '*multifd-qatzip-level': 'uint8', > >> + '*multifd-qatzip-sw-fallback': 'bool', > >> '*multifd-zstd-level': 'uint8', > >> '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], > >> '*x-vcpu-dirty-limit-period': { 'type': 'uint64', > >> @@ -1273,6 +1296,16 @@ > >> # speed, and 9 means best compression ratio which will consume > >> # more CPU. Defaults to 1. (Since 5.0) > >> # > >> +# @multifd-qatzip-level: Set the compression level to be used in live > >> +# migration. The level is an integer between 1 and 9, where 1 > means > >> +# the best compression speed, and 9 means the best compression > >> +# ratio which will consume more CPU. Defaults to 1. > >> +# > >> +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT > hardware > >> +# is unavailable. Defaults to false. Software fallback performance > >> +# is very poor compared to regular zlib, so be cautious about > >> +# enabling this option. > >> +# > >> # @multifd-zstd-level: Set the compression level to be used in live > >> # migration, the compression level is an integer between 0 and 20, > >> # where 0 means no compression, 1 means the best compression > >> @@ -1350,6 +1383,8 @@ > >> '*max-cpu-throttle': 'uint8', > >> '*multifd-compression': 'MultiFDCompression', > >> '*multifd-zlib-level': 'uint8', > >> + '*multifd-qatzip-level': 'uint8', > >> + '*multifd-qatzip-sw-fallback': 'bool', > >> '*multifd-zstd-level': 'uint8', > >> '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], > >> '*x-vcpu-dirty-limit-period': { 'type': 'uint64', > >> -- > >> 2.30.2 >
diff --git a/migration/migration-hmp-cmds.c b/migration/migration-hmp-cmds.c index 99b49df5dd..4bd23ba14d 100644 --- a/migration/migration-hmp-cmds.c +++ b/migration/migration-hmp-cmds.c @@ -630,6 +630,14 @@ void hmp_migrate_set_parameter(Monitor *mon, const QDict *qdict) p->has_multifd_zlib_level = true; visit_type_uint8(v, param, &p->multifd_zlib_level, &err); break; + case MIGRATION_PARAMETER_MULTIFD_QATZIP_LEVEL: + p->has_multifd_qatzip_level = true; + visit_type_uint8(v, param, &p->multifd_qatzip_level, &err); + break; + case MIGRATION_PARAMETER_MULTIFD_QATZIP_SW_FALLBACK: + p->has_multifd_qatzip_sw_fallback = true; + visit_type_bool(v, param, &p->multifd_qatzip_sw_fallback, &err); + break; case MIGRATION_PARAMETER_MULTIFD_ZSTD_LEVEL: p->has_multifd_zstd_level = true; visit_type_uint8(v, param, &p->multifd_zstd_level, &err); diff --git a/migration/options.c b/migration/options.c index 3e3e0b93b4..1316ea605a 100644 --- a/migration/options.c +++ b/migration/options.c @@ -62,6 +62,15 @@ #define DEFAULT_MIGRATE_MULTIFD_COMPRESSION MULTIFD_COMPRESSION_NONE /* 0: means nocompress, 1: best speed, ... 9: best compress ratio */ #define DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL 1 +/* + * 1: best speed, ... 9: best compress ratio + * There is some nuance here. Refer to QATzip documentation to understand + * the mapping of QATzip levels to standard deflate levels. + */ +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL 1 +/* QATzip's SW fallback implementation is extremely slow, so avoid fallback */ +#define DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK false + /* 0: means nocompress, 1: best speed, ... 20: best compress ratio */ #define DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL 1 @@ -143,6 +152,12 @@ Property migration_properties[] = { DEFINE_PROP_UINT8("multifd-zlib-level", MigrationState, parameters.multifd_zlib_level, DEFAULT_MIGRATE_MULTIFD_ZLIB_LEVEL), + DEFINE_PROP_UINT8("multifd-qatzip-level", MigrationState, + parameters.multifd_qatzip_level, + DEFAULT_MIGRATE_MULTIFD_QATZIP_LEVEL), + DEFINE_PROP_BOOL("multifd-qatzip-sw-fallback", MigrationState, + parameters.multifd_qatzip_sw_fallback, + DEFAULT_MIGRATE_MULTIFD_QATZIP_SW_FALLBACK), DEFINE_PROP_UINT8("multifd-zstd-level", MigrationState, parameters.multifd_zstd_level, DEFAULT_MIGRATE_MULTIFD_ZSTD_LEVEL), @@ -861,6 +876,20 @@ int migrate_multifd_zlib_level(void) return s->parameters.multifd_zlib_level; } +int migrate_multifd_qatzip_level(void) +{ + MigrationState *s = migrate_get_current(); + + return s->parameters.multifd_qatzip_level; +} + +bool migrate_multifd_qatzip_sw_fallback(void) +{ + MigrationState *s = migrate_get_current(); + + return s->parameters.multifd_qatzip_sw_fallback; +} + int migrate_multifd_zstd_level(void) { MigrationState *s = migrate_get_current(); @@ -983,6 +1012,11 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) params->multifd_compression = s->parameters.multifd_compression; params->has_multifd_zlib_level = true; params->multifd_zlib_level = s->parameters.multifd_zlib_level; + params->has_multifd_qatzip_level = true; + params->multifd_qatzip_level = s->parameters.multifd_qatzip_level; + params->has_multifd_qatzip_sw_fallback = true; + params->multifd_qatzip_sw_fallback = + s->parameters.multifd_qatzip_sw_fallback; params->has_multifd_zstd_level = true; params->multifd_zstd_level = s->parameters.multifd_zstd_level; params->has_xbzrle_cache_size = true; @@ -1038,6 +1072,8 @@ void migrate_params_init(MigrationParameters *params) params->has_multifd_channels = true; params->has_multifd_compression = true; params->has_multifd_zlib_level = true; + params->has_multifd_qatzip_level = true; + params->has_multifd_qatzip_sw_fallback = true; params->has_multifd_zstd_level = true; params->has_xbzrle_cache_size = true; params->has_max_postcopy_bandwidth = true; @@ -1147,6 +1183,14 @@ bool migrate_params_check(MigrationParameters *params, Error **errp) return false; } + if (params->has_multifd_qatzip_level && + ((params->multifd_qatzip_level > 9) || + (params->multifd_qatzip_level < 1))) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_qatzip_level", + "a value between 1 and 9"); + return false; + } + if (params->has_multifd_zstd_level && (params->multifd_zstd_level > 20)) { error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "multifd_zstd_level", @@ -1312,6 +1356,12 @@ static void migrate_params_test_apply(MigrateSetParameters *params, if (params->has_multifd_compression) { dest->multifd_compression = params->multifd_compression; } + if (params->has_multifd_qatzip_level) { + dest->multifd_qatzip_level = params->multifd_qatzip_level; + } + if (params->has_multifd_qatzip_sw_fallback) { + dest->multifd_qatzip_sw_fallback = params->multifd_qatzip_sw_fallback; + } if (params->has_xbzrle_cache_size) { dest->xbzrle_cache_size = params->xbzrle_cache_size; } @@ -1447,6 +1497,13 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) if (params->has_multifd_compression) { s->parameters.multifd_compression = params->multifd_compression; } + if (params->has_multifd_qatzip_level) { + s->parameters.multifd_qatzip_level = params->multifd_qatzip_level; + } + if (params->has_multifd_qatzip_sw_fallback) { + s->parameters.multifd_qatzip_sw_fallback = + params->multifd_qatzip_sw_fallback; + } if (params->has_xbzrle_cache_size) { s->parameters.xbzrle_cache_size = params->xbzrle_cache_size; xbzrle_cache_resize(params->xbzrle_cache_size, errp); diff --git a/migration/options.h b/migration/options.h index 246c160aee..94aee24d97 100644 --- a/migration/options.h +++ b/migration/options.h @@ -87,6 +87,8 @@ MigMode migrate_mode(void); int migrate_multifd_channels(void); MultiFDCompression migrate_multifd_compression(void); int migrate_multifd_zlib_level(void); +int migrate_multifd_qatzip_level(void); +bool migrate_multifd_qatzip_sw_fallback(void); int migrate_multifd_zstd_level(void); uint8_t migrate_throttle_trigger_threshold(void); const char *migrate_tls_authz(void); diff --git a/qapi/migration.json b/qapi/migration.json index 0b33a71ab4..66ea6d32fc 100644 --- a/qapi/migration.json +++ b/qapi/migration.json @@ -853,6 +853,16 @@ # speed, and 9 means best compression ratio which will consume # more CPU. Defaults to 1. (Since 5.0) # +# @multifd-qatzip-level: Set the compression level to be used in live +# migration. The level is an integer between 1 and 9, where 1 means +# the best compression speed, and 9 means the best compression +# ratio which will consume more CPU. Defaults to 1. +# +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware +# is unavailable. Defaults to false. Software fallback performance +# is very poor compared to regular zlib, so be cautious about +# enabling this option. +# # @multifd-zstd-level: Set the compression level to be used in live # migration, the compression level is an integer between 0 and 20, # where 0 means no compression, 1 means the best compression @@ -915,6 +925,7 @@ 'xbzrle-cache-size', 'max-postcopy-bandwidth', 'max-cpu-throttle', 'multifd-compression', 'multifd-zlib-level', 'multifd-zstd-level', + 'multifd-qatzip-level', 'multifd-qatzip-sw-fallback', 'block-bitmap-mapping', { 'name': 'x-vcpu-dirty-limit-period', 'features': ['unstable'] }, 'vcpu-dirty-limit', @@ -1045,6 +1056,16 @@ # speed, and 9 means best compression ratio which will consume # more CPU. Defaults to 1. (Since 5.0) # +# @multifd-qatzip-level: Set the compression level to be used in live +# migration. The level is an integer between 1 and 9, where 1 means +# the best compression speed, and 9 means the best compression +# ratio which will consume more CPU. Defaults to 1. +# +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware +# is unavailable. Defaults to false. Software fallback performance +# is very poor compared to regular zlib, so be cautious about +# enabling this option. +# # @multifd-zstd-level: Set the compression level to be used in live # migration, the compression level is an integer between 0 and 20, # where 0 means no compression, 1 means the best compression @@ -1125,6 +1146,8 @@ '*max-cpu-throttle': 'uint8', '*multifd-compression': 'MultiFDCompression', '*multifd-zlib-level': 'uint8', + '*multifd-qatzip-level': 'uint8', + '*multifd-qatzip-sw-fallback': 'bool', '*multifd-zstd-level': 'uint8', '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], '*x-vcpu-dirty-limit-period': { 'type': 'uint64', @@ -1273,6 +1296,16 @@ # speed, and 9 means best compression ratio which will consume # more CPU. Defaults to 1. (Since 5.0) # +# @multifd-qatzip-level: Set the compression level to be used in live +# migration. The level is an integer between 1 and 9, where 1 means +# the best compression speed, and 9 means the best compression +# ratio which will consume more CPU. Defaults to 1. +# +# @multifd-qatzip-sw-fallback: Enable software fallback if QAT hardware +# is unavailable. Defaults to false. Software fallback performance +# is very poor compared to regular zlib, so be cautious about +# enabling this option. +# # @multifd-zstd-level: Set the compression level to be used in live # migration, the compression level is an integer between 0 and 20, # where 0 means no compression, 1 means the best compression @@ -1350,6 +1383,8 @@ '*max-cpu-throttle': 'uint8', '*multifd-compression': 'MultiFDCompression', '*multifd-zlib-level': 'uint8', + '*multifd-qatzip-level': 'uint8', + '*multifd-qatzip-sw-fallback': 'bool', '*multifd-zstd-level': 'uint8', '*block-bitmap-mapping': [ 'BitmapMigrationNodeAlias' ], '*x-vcpu-dirty-limit-period': { 'type': 'uint64',