| Message ID | 20240730170347.4103919-6-berrange@redhat.com |
|---|---|
| State | New |
| Headers | show |
| Series | Convert avocado tests to normal Python unittests | expand |
Hi, On 30/7/24 19:03, Daniel P. Berrangé wrote: > From: Thomas Huth <thuth@redhat.com> > > Provide a meson.build file for the upcoming python-based functional > tests, and add some wrapper glue targets to the tests/Makefile.include > file. We are going to use two "speed" modes for the functional tests: > The "quick" tests can be run at any time (i.e. also during "make check"), > while the "thorough" tests should only be run when running a > "make check-functional" test run (since these tests might download > additional assets from the internet). > > The changes to the meson.build files are partly based on an earlier > patch by Ani Sinha. > > Signed-off-by: Thomas Huth <thuth@redhat.com> > --- > tests/Makefile.include | 11 ++++++ > tests/functional/meson.build | 66 ++++++++++++++++++++++++++++++++++++ > tests/meson.build | 1 + > 3 files changed, 78 insertions(+) > create mode 100644 tests/functional/meson.build > +foreach speed : ['quick', 'thorough'] > + foreach dir : target_dirs > + if not dir.endswith('-softmmu') > + continue > + endif > + > + target_base = dir.split('-')[0] > + test_emulator = emulators['qemu-system-' + target_base] > + > + if speed == 'quick' > + suites = ['func-quick', 'func-' + target_base] > + target_tests = get_variable('tests_' + target_base + '_quick', []) + tests_generic > + else > + suites = ['func-' + speed, 'func-' + target_base + '-' + speed, speed] > + target_tests = get_variable('tests_' + target_base + '_' + speed, []) > + endif > + > + test_deps = roms > + test_env = environment() > + if have_tools > + test_env.set('QEMU_TEST_QEMU_IMG', meson.global_build_root() / 'qemu-img') > + test_deps += [qemu_img] > + endif > + test_env.set('QEMU_TEST_QEMU_BINARY', > + meson.global_build_root() / 'qemu-system-' + target_base) > + test_env.set('QEMU_BUILD_ROOT', meson.project_build_root()) > + test_env.set('PYTHONPATH', meson.project_source_root() / 'python:' + > + meson.current_source_dir()) > + > + foreach test : target_tests > + test('func-@0@/@1@'.format(target_base, test), > + python, > + depends: [test_deps, test_emulator, emulator_modules], > + env: test_env, > + args: [meson.current_source_dir() / 'test_' + test + '.py'], > + protocol: 'tap', > + timeout: test_timeouts.get(test, 60), > + priority: test_timeouts.get(test, 60), > + suite: suites) > + endforeach > + endforeach > +endforeach Can we have meson report why a test is skipped? All Python unittest @skip* decorators provide a 'reason' argument, so we already have that information. Not much help on meson doc: https://mesonbuild.com/Unit-tests.html#skipped-tests-and-hard-errors If I run a test manually I see the reason in the tap output: $ FOO=bar tests/functional/test_mips64el_loongson3v.py TAP version 13 ok 1 test_mips64el_loongson3v.MipsLoongson3v.test_pmon_serial_console # SKIP untrusted code 1..1 $ Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Tested-by: Philippe Mathieu-Daudé <philmd@linaro.org>
On 01/08/2024 18.12, Philippe Mathieu-Daudé wrote: > Hi, > > On 30/7/24 19:03, Daniel P. Berrangé wrote: >> From: Thomas Huth <thuth@redhat.com> >> >> Provide a meson.build file for the upcoming python-based functional >> tests, and add some wrapper glue targets to the tests/Makefile.include >> file. We are going to use two "speed" modes for the functional tests: >> The "quick" tests can be run at any time (i.e. also during "make check"), >> while the "thorough" tests should only be run when running a >> "make check-functional" test run (since these tests might download >> additional assets from the internet). >> >> The changes to the meson.build files are partly based on an earlier >> patch by Ani Sinha. >> >> Signed-off-by: Thomas Huth <thuth@redhat.com> >> --- >> tests/Makefile.include | 11 ++++++ >> tests/functional/meson.build | 66 ++++++++++++++++++++++++++++++++++++ >> tests/meson.build | 1 + >> 3 files changed, 78 insertions(+) >> create mode 100644 tests/functional/meson.build > > >> +foreach speed : ['quick', 'thorough'] >> + foreach dir : target_dirs >> + if not dir.endswith('-softmmu') >> + continue >> + endif >> + >> + target_base = dir.split('-')[0] >> + test_emulator = emulators['qemu-system-' + target_base] >> + >> + if speed == 'quick' >> + suites = ['func-quick', 'func-' + target_base] >> + target_tests = get_variable('tests_' + target_base + '_quick', []) >> + tests_generic >> + else >> + suites = ['func-' + speed, 'func-' + target_base + '-' + speed, speed] >> + target_tests = get_variable('tests_' + target_base + '_' + speed, []) >> + endif >> + >> + test_deps = roms >> + test_env = environment() >> + if have_tools >> + test_env.set('QEMU_TEST_QEMU_IMG', meson.global_build_root() / >> 'qemu-img') >> + test_deps += [qemu_img] >> + endif >> + test_env.set('QEMU_TEST_QEMU_BINARY', >> + meson.global_build_root() / 'qemu-system-' + target_base) >> + test_env.set('QEMU_BUILD_ROOT', meson.project_build_root()) >> + test_env.set('PYTHONPATH', meson.project_source_root() / 'python:' + >> + meson.current_source_dir()) >> + >> + foreach test : target_tests >> + test('func-@0@/@1@'.format(target_base, test), >> + python, >> + depends: [test_deps, test_emulator, emulator_modules], >> + env: test_env, >> + args: [meson.current_source_dir() / 'test_' + test + '.py'], >> + protocol: 'tap', >> + timeout: test_timeouts.get(test, 60), >> + priority: test_timeouts.get(test, 60), >> + suite: suites) >> + endforeach >> + endforeach >> +endforeach > > Can we have meson report why a test is skipped? All Python > unittest @skip* decorators provide a 'reason' argument, so > we already have that information. I guess you'd need to extend the meson test runner for this... > Not much help on meson doc: > https://mesonbuild.com/Unit-tests.html#skipped-tests-and-hard-errors > > If I run a test manually I see the reason in the tap output: > > $ FOO=bar tests/functional/test_mips64el_loongson3v.py > TAP version 13 > ok 1 test_mips64el_loongson3v.MipsLoongson3v.test_pmon_serial_console # SKIP > untrusted code > 1..1 You can also see this information in the txt file in the meson-logs folder, so it is certainly not completely lost. Thomas
diff --git a/tests/Makefile.include b/tests/Makefile.include index 167d8f29a4..d0c2ae30b7 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -9,6 +9,8 @@ check-help: @echo "Individual test suites:" @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" @echo " $(MAKE) check-qtest Run qtest tests" + @echo " $(MAKE) check-functional Run python-based functional tests" + @echo " $(MAKE) check-functional-TARGET Run functional tests for a given target" @echo " $(MAKE) check-unit Run qobject tests" @echo " $(MAKE) check-qapi-schema Run QAPI schema tests" @echo " $(MAKE) check-block Run block tests" @@ -152,6 +154,15 @@ check-acceptance-deprecated-warning: check-acceptance: check-acceptance-deprecated-warning | check-avocado +FUNCTIONAL_TARGETS=$(patsubst %-softmmu,check-functional-%, $(filter %-softmmu,$(TARGETS))) +.PHONY: $(FUNCTIONAL_TARGETS) +$(FUNCTIONAL_TARGETS): + @$(MAKE) SPEED=thorough $(subst -functional,-func,$@) + +.PHONY: check-functional +check-functional: + @$(MAKE) SPEED=thorough check-func check-func-quick + # Consolidated targets .PHONY: check check-clean get-vm-images diff --git a/tests/functional/meson.build b/tests/functional/meson.build new file mode 100644 index 0000000000..7fc9c4e513 --- /dev/null +++ b/tests/functional/meson.build @@ -0,0 +1,66 @@ +# QEMU functional tests: +# Tests that are put in the 'quick' category are run by default during +# 'make check'. Everything that should not be run during 'make check' +# (e.g. tests that fetch assets from the internet) should be put into +# the 'thorough' category instead. + +# Most tests run too slow with TCI enabled, so skip the functional tests there +if get_option('tcg_interpreter') + subdir_done() +endif + +# Timeouts for individual tests that can be slow e.g. with debugging enabled +test_timeouts = { +} + +tests_generic = [ +] + +tests_x86_64_quick = [ +] + +tests_x86_64_thorough = [ +] + +foreach speed : ['quick', 'thorough'] + foreach dir : target_dirs + if not dir.endswith('-softmmu') + continue + endif + + target_base = dir.split('-')[0] + test_emulator = emulators['qemu-system-' + target_base] + + if speed == 'quick' + suites = ['func-quick', 'func-' + target_base] + target_tests = get_variable('tests_' + target_base + '_quick', []) + tests_generic + else + suites = ['func-' + speed, 'func-' + target_base + '-' + speed, speed] + target_tests = get_variable('tests_' + target_base + '_' + speed, []) + endif + + test_deps = roms + test_env = environment() + if have_tools + test_env.set('QEMU_TEST_QEMU_IMG', meson.global_build_root() / 'qemu-img') + test_deps += [qemu_img] + endif + test_env.set('QEMU_TEST_QEMU_BINARY', + meson.global_build_root() / 'qemu-system-' + target_base) + test_env.set('QEMU_BUILD_ROOT', meson.project_build_root()) + test_env.set('PYTHONPATH', meson.project_source_root() / 'python:' + + meson.current_source_dir()) + + foreach test : target_tests + test('func-@0@/@1@'.format(target_base, test), + python, + depends: [test_deps, test_emulator, emulator_modules], + env: test_env, + args: [meson.current_source_dir() / 'test_' + test + '.py'], + protocol: 'tap', + timeout: test_timeouts.get(test, 60), + priority: test_timeouts.get(test, 60), + suite: suites) + endforeach + endforeach +endforeach diff --git a/tests/meson.build b/tests/meson.build index acb6807094..3345ad2098 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -85,3 +85,4 @@ subdir('unit') subdir('qapi-schema') subdir('qtest') subdir('migration') +subdir('functional')