| Message ID | 20240711115546.40859-5-thuth@redhat.com |
|---|---|
| State | New |
| Headers | show |
| Series | Convert avocado tests to normal Python unittests | expand |
On Thu, Jul 11, 2024 at 01:55:42PM +0200, Thomas Huth wrote: > From: Ani Sinha <ani@anisinha.ca> > > Integrate the pytest framework with the meson build system. This > will make meson run all the pytests under the pytest directory. Lets add a note about the compelling benefit of this new approach With this change, each functional test becomes subject to an individual execution timeout, defaulting to 60 seconds, but overridable per-test. > > Signed-off-by: Ani Sinha <ani@anisinha.ca> > [thuth: Removed the acpi-bits and adjusted for converted avocado tests instead] > Signed-off-by: Thomas Huth <thuth@redhat.com> > --- > tests/Makefile.include | 4 ++- > tests/meson.build | 1 + > tests/pytest/meson.build | 53 ++++++++++++++++++++++++++++++++++++++++ > 3 files changed, 57 insertions(+), 1 deletion(-) > create mode 100644 tests/pytest/meson.build For CI purposes we'll need to add 'python3-pytest' to tests/lcitool/projects/qemu.yml, and re-generate the the dockerfiles. Some of the other non-gitlab CI integrations probably need manual additions of pytest packages. > diff --git a/tests/Makefile.include b/tests/Makefile.include > index d39d5dd6a4..68151717d7 100644 > --- a/tests/Makefile.include > +++ b/tests/Makefile.include > @@ -3,12 +3,14 @@ > .PHONY: check-help > check-help: > @echo "Regression testing targets:" > - @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests" > + @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest, pytest and decodetree tests" > @echo " $(MAKE) bench Run speed tests" > @echo > @echo "Individual test suites:" > @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" > @echo " $(MAKE) check-qtest Run qtest tests" > + @echo " $(MAKE) check-pytest Run pytest tests" > + @echo " $(MAKE) check-pytest-TARGET Run pytest for a given target" Or name it after the type of test rather than harness ? eg check-functional / check-functional-TARGET For that matter perhaps also for the dir name ? tests/functional/*.py > @echo " $(MAKE) check-unit Run qobject tests" > @echo " $(MAKE) check-qapi-schema Run QAPI schema tests" > @echo " $(MAKE) check-block Run block tests" > diff --git a/tests/meson.build b/tests/meson.build > index acb6807094..17510a468e 100644 > --- a/tests/meson.build > +++ b/tests/meson.build > @@ -85,3 +85,4 @@ subdir('unit') > subdir('qapi-schema') > subdir('qtest') > subdir('migration') > +subdir('pytest') > diff --git a/tests/pytest/meson.build b/tests/pytest/meson.build > new file mode 100644 > index 0000000000..1486628d45 > --- /dev/null > +++ b/tests/pytest/meson.build > @@ -0,0 +1,53 @@ > +slow_pytests = { > + 'mem_addr_space' : 90, > +} > + > +pytests_generic = [ > + 'empty_cpu_model', > + 'info_usernet', > + 'version', > +] > + > +pytests_x86_64 = [ > + 'cpu_queries', > + 'mem_addr_space', > + 'virtio_version', > +] > + > +pytest = find_program('pytest', required: false) > +if not pytest.found() > + message('pytest not available ==> Disabled the qemu-pytests.') > + subdir_done() > +endif > + > +foreach dir : target_dirs > + if not dir.endswith('-softmmu') > + continue > + endif > + > + target_base = dir.split('-')[0] > + pytest_emulator = emulators['qemu-system-' + target_base] > + target_pytests = get_variable('pytests_' + target_base, []) + pytests_generic > + > + test_deps = roms > + pytest_env = environment() > + if have_tools > + pytest_env.set('PYTEST_QEMU_IMG', './qemu-img') > + test_deps += [qemu_img] > + endif > + pytest_env.set('PYTEST_QEMU_BINARY', meson.global_build_root() / 'qemu-system-' + target_base) > + pytest_env.set('PYTEST_SOURCE_ROOT', meson.project_source_root()) > + pytest_env.set('PYTEST_BUILD_ROOT', meson.project_build_root()) > + pytest_env.set('PYTHONPATH', meson.project_source_root() / 'python') > + > + foreach test : target_pytests > + test('pytest-@0@/@1@'.format(target_base, test), > + pytest, > + depends: [test_deps, pytest_emulator, emulator_modules], > + env: pytest_env, > + args: [meson.current_source_dir() / 'test_' + test + '.py'], > + timeout: slow_pytests.get(test, 60), > + priority: slow_pytests.get(test, 60), > + suite: ['pytest', 'pytest-' + target_base]) > + endforeach > +endforeach > -- > 2.45.2 > With regards, Daniel
On 12/07/2024 11.01, Daniel P. Berrangé wrote: > On Thu, Jul 11, 2024 at 01:55:42PM +0200, Thomas Huth wrote: >> From: Ani Sinha <ani@anisinha.ca> >> >> Integrate the pytest framework with the meson build system. This >> will make meson run all the pytests under the pytest directory. > > Lets add a note about the compelling benefit of this new approach > > With this change, each functional test becomes subject > to an individual execution timeout, defaulting to 60 > seconds, but overridable per-test. The avocado runner uses timeouts, too, so it's not really an additional benefit that we get here. > For CI purposes we'll need to add 'python3-pytest' to > tests/lcitool/projects/qemu.yml, and re-generate the > the dockerfiles. Some of the other non-gitlab CI > integrations probably need manual additions of pytest > packages. I'm currently rather looking into getting rid of pytest and to use pycotap instead: Using the TAP protocol for running the tests, you get a much nicer output from the meson test runner, which can then count the subtests and properly report SKIPs for tests that have not been run. >> diff --git a/tests/Makefile.include b/tests/Makefile.include >> index d39d5dd6a4..68151717d7 100644 >> --- a/tests/Makefile.include >> +++ b/tests/Makefile.include >> @@ -3,12 +3,14 @@ >> .PHONY: check-help >> check-help: >> @echo "Regression testing targets:" >> - @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests" >> + @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest, pytest and decodetree tests" >> @echo " $(MAKE) bench Run speed tests" >> @echo >> @echo "Individual test suites:" >> @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" >> @echo " $(MAKE) check-qtest Run qtest tests" >> + @echo " $(MAKE) check-pytest Run pytest tests" >> + @echo " $(MAKE) check-pytest-TARGET Run pytest for a given target" > > Or name it after the type of test rather than harness ? > > eg check-functional / check-functional-TARGET > > For that matter perhaps also for the dir name ? > > tests/functional/*.py I almost expected that discussion again ... (see https://lists.gnu.org/archive/html/qemu-devel/2021-05/msg06553.html ) ... last time we couldn't really agree on such a name and decided to go with the name of the framework... I agree that "pytest" is likely not the best name here, especially if switching to the pycotap test runner instead of using the "pytest" program, but "functional" might trigger the same discussion again as last time ... should it rather be "functional" or "validation" or "integration" etc.? Maybe best if we come up with a new fictional name for the "new" test framework... something like "pyqe" - PYthon-based Qemu test Environment"? ... could be considered as a play on the word "pike", too, i.e. something that makes sure that not everything gets in ... ? WDYT? Thomas
On Fri, Jul 12, 2024 at 12:14:45PM +0200, Thomas Huth wrote: > On 12/07/2024 11.01, Daniel P. Berrangé wrote: > > On Thu, Jul 11, 2024 at 01:55:42PM +0200, Thomas Huth wrote: > > > From: Ani Sinha <ani@anisinha.ca> > > > > > > Integrate the pytest framework with the meson build system. This > > > will make meson run all the pytests under the pytest directory. > > > > Lets add a note about the compelling benefit of this new approach > > > > With this change, each functional test becomes subject > > to an individual execution timeout, defaulting to 60 > > seconds, but overridable per-test. > > The avocado runner uses timeouts, too, so it's not really an additional > benefit that we get here. At the meson level though, we can't put an overall cap on the execution time, as there's only 1 huge test visible, and thus the meson timeout multiplier also won't get honoured IIUC. > > > For CI purposes we'll need to add 'python3-pytest' to > > tests/lcitool/projects/qemu.yml, and re-generate the > > the dockerfiles. Some of the other non-gitlab CI > > integrations probably need manual additions of pytest > > packages. > > I'm currently rather looking into getting rid of pytest and to use pycotap > instead: Using the TAP protocol for running the tests, you get a much nicer > output from the meson test runner, which can then count the subtests and > properly report SKIPs for tests that have not been run. > > > > diff --git a/tests/Makefile.include b/tests/Makefile.include > > > index d39d5dd6a4..68151717d7 100644 > > > --- a/tests/Makefile.include > > > +++ b/tests/Makefile.include > > > @@ -3,12 +3,14 @@ > > > .PHONY: check-help > > > check-help: > > > @echo "Regression testing targets:" > > > - @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests" > > > + @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest, pytest and decodetree tests" > > > @echo " $(MAKE) bench Run speed tests" > > > @echo > > > @echo "Individual test suites:" > > > @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" > > > @echo " $(MAKE) check-qtest Run qtest tests" > > > + @echo " $(MAKE) check-pytest Run pytest tests" > > > + @echo " $(MAKE) check-pytest-TARGET Run pytest for a given target" > > > > Or name it after the type of test rather than harness ? > > > > eg check-functional / check-functional-TARGET > > > > For that matter perhaps also for the dir name ? > > > > tests/functional/*.py > > I almost expected that discussion again ... (see > https://lists.gnu.org/archive/html/qemu-devel/2021-05/msg06553.html ) ... > last time we couldn't really agree on such a name and decided to go with the > name of the framework... > > I agree that "pytest" is likely not the best name here, especially if > switching to the pycotap test runner instead of using the "pytest" program, > but "functional" might trigger the same discussion again as last time ... > should it rather be "functional" or "validation" or "integration" etc.? IMHO you can just make an executive decision and pick one of those three. None of them are terrible, any would be a valid choice. > Maybe best if we come up with a new fictional name for the "new" test > framework... something like "pyqe" - PYthon-based Qemu test Environment"? > ... could be considered as a play on the word "pike", too, i.e. something > that makes sure that not everything gets in ... ? WDYT? A wierd acronym isn't really going to tell contributors anything about what this is, compared to calling it 'functional' or 'integration', etc. With regards, Daniel
On Fri, Jul 12, 2024 at 12:14:45PM +0200, Thomas Huth wrote: > On 12/07/2024 11.01, Daniel P. Berrangé wrote: > > On Thu, Jul 11, 2024 at 01:55:42PM +0200, Thomas Huth wrote: > > > From: Ani Sinha <ani@anisinha.ca> > > > > > > Integrate the pytest framework with the meson build system. This > > > will make meson run all the pytests under the pytest directory. > > > > Lets add a note about the compelling benefit of this new approach > > > > With this change, each functional test becomes subject > > to an individual execution timeout, defaulting to 60 > > seconds, but overridable per-test. > > The avocado runner uses timeouts, too, so it's not really an additional > benefit that we get here. > > > For CI purposes we'll need to add 'python3-pytest' to > > tests/lcitool/projects/qemu.yml, and re-generate the > > the dockerfiles. Some of the other non-gitlab CI > > integrations probably need manual additions of pytest > > packages. > > I'm currently rather looking into getting rid of pytest and to use pycotap > instead: Using the TAP protocol for running the tests, you get a much nicer > output from the meson test runner, which can then count the subtests and > properly report SKIPs for tests that have not been run. I've just looked at pycotap and IIUC, there's no command line tool equivalent to '/usr/bin/pytest' at all. Each test case is expected to provide a stub for "__main__" to invoke the tests. As such each individual test is directly executable. This meshes nicely with what I'd suggested as changes in patch 1, and eliminating the intermediate runner process is a nice further simplification. So I'll be interested to see your next version using pycotap. With regards, Daniel
On 12/07/2024 12.26, Daniel P. Berrangé wrote: > On Fri, Jul 12, 2024 at 12:14:45PM +0200, Thomas Huth wrote: >> On 12/07/2024 11.01, Daniel P. Berrangé wrote: >>> On Thu, Jul 11, 2024 at 01:55:42PM +0200, Thomas Huth wrote: >>>> From: Ani Sinha <ani@anisinha.ca> >>>> >>>> Integrate the pytest framework with the meson build system. This >>>> will make meson run all the pytests under the pytest directory. >>> >>> Lets add a note about the compelling benefit of this new approach >>> >>> With this change, each functional test becomes subject >>> to an individual execution timeout, defaulting to 60 >>> seconds, but overridable per-test. >> >> The avocado runner uses timeouts, too, so it's not really an additional >> benefit that we get here. > > At the meson level though, we can't put an overall cap on > the execution time, as there's only 1 huge test visible, > and thus the meson timeout multiplier also won't get > honoured IIUC. > >> >>> For CI purposes we'll need to add 'python3-pytest' to >>> tests/lcitool/projects/qemu.yml, and re-generate the >>> the dockerfiles. Some of the other non-gitlab CI >>> integrations probably need manual additions of pytest >>> packages. >> >> I'm currently rather looking into getting rid of pytest and to use pycotap >> instead: Using the TAP protocol for running the tests, you get a much nicer >> output from the meson test runner, which can then count the subtests and >> properly report SKIPs for tests that have not been run. >> >>>> diff --git a/tests/Makefile.include b/tests/Makefile.include >>>> index d39d5dd6a4..68151717d7 100644 >>>> --- a/tests/Makefile.include >>>> +++ b/tests/Makefile.include >>>> @@ -3,12 +3,14 @@ >>>> .PHONY: check-help >>>> check-help: >>>> @echo "Regression testing targets:" >>>> - @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests" >>>> + @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest, pytest and decodetree tests" >>>> @echo " $(MAKE) bench Run speed tests" >>>> @echo >>>> @echo "Individual test suites:" >>>> @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" >>>> @echo " $(MAKE) check-qtest Run qtest tests" >>>> + @echo " $(MAKE) check-pytest Run pytest tests" >>>> + @echo " $(MAKE) check-pytest-TARGET Run pytest for a given target" >>> >>> Or name it after the type of test rather than harness ? >>> >>> eg check-functional / check-functional-TARGET >>> >>> For that matter perhaps also for the dir name ? >>> >>> tests/functional/*.py >> >> I almost expected that discussion again ... (see >> https://lists.gnu.org/archive/html/qemu-devel/2021-05/msg06553.html ) ... >> last time we couldn't really agree on such a name and decided to go with the >> name of the framework... >> >> I agree that "pytest" is likely not the best name here, especially if >> switching to the pycotap test runner instead of using the "pytest" program, >> but "functional" might trigger the same discussion again as last time ... >> should it rather be "functional" or "validation" or "integration" etc.? > > IMHO you can just make an executive decision and pick one of those > three. None of them are terrible, any would be a valid choice. Ok, I think I'll go with "functional" since that still seems to be the best fit to me. Thomas
On 12/07/2024 13.47, Daniel P. Berrangé wrote: > On Fri, Jul 12, 2024 at 12:14:45PM +0200, Thomas Huth wrote: >> On 12/07/2024 11.01, Daniel P. Berrangé wrote: >>> On Thu, Jul 11, 2024 at 01:55:42PM +0200, Thomas Huth wrote: >>>> From: Ani Sinha <ani@anisinha.ca> >>>> >>>> Integrate the pytest framework with the meson build system. This >>>> will make meson run all the pytests under the pytest directory. >>> >>> Lets add a note about the compelling benefit of this new approach >>> >>> With this change, each functional test becomes subject >>> to an individual execution timeout, defaulting to 60 >>> seconds, but overridable per-test. >> >> The avocado runner uses timeouts, too, so it's not really an additional >> benefit that we get here. >> >>> For CI purposes we'll need to add 'python3-pytest' to >>> tests/lcitool/projects/qemu.yml, and re-generate the >>> the dockerfiles. Some of the other non-gitlab CI >>> integrations probably need manual additions of pytest >>> packages. >> >> I'm currently rather looking into getting rid of pytest and to use pycotap >> instead: Using the TAP protocol for running the tests, you get a much nicer >> output from the meson test runner, which can then count the subtests and >> properly report SKIPs for tests that have not been run. > > I've just looked at pycotap and IIUC, there's no command line > tool equivalent to '/usr/bin/pytest' at all. Each test case > is expected to provide a stub for "__main__" to invoke the > tests. With pycotap 1.3.0 you could run tests through "python -m pycotap ...", too. But for playing nicely with meson, I'd like to set message_log and test_output_log to LogMode.LogToError instead of using their default value, so I need my own wrapper code anyway. > As such each individual test is directly executable. > This meshes nicely with what I'd suggested as changes in > patch 1, and eliminating the intermediate runner process is > a nice further simplification. So I'll be interested to see > your next version using pycotap. I'll try to post it as soon as I have something at least in a somewhat more proper shape than the RFC ;-) Thomas
diff --git a/tests/Makefile.include b/tests/Makefile.include index d39d5dd6a4..68151717d7 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -3,12 +3,14 @@ .PHONY: check-help check-help: @echo "Regression testing targets:" - @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests" + @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest, pytest and decodetree tests" @echo " $(MAKE) bench Run speed tests" @echo @echo "Individual test suites:" @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" @echo " $(MAKE) check-qtest Run qtest tests" + @echo " $(MAKE) check-pytest Run pytest tests" + @echo " $(MAKE) check-pytest-TARGET Run pytest for a given target" @echo " $(MAKE) check-unit Run qobject tests" @echo " $(MAKE) check-qapi-schema Run QAPI schema tests" @echo " $(MAKE) check-block Run block tests" diff --git a/tests/meson.build b/tests/meson.build index acb6807094..17510a468e 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -85,3 +85,4 @@ subdir('unit') subdir('qapi-schema') subdir('qtest') subdir('migration') +subdir('pytest') diff --git a/tests/pytest/meson.build b/tests/pytest/meson.build new file mode 100644 index 0000000000..1486628d45 --- /dev/null +++ b/tests/pytest/meson.build @@ -0,0 +1,53 @@ +slow_pytests = { + 'mem_addr_space' : 90, +} + +pytests_generic = [ + 'empty_cpu_model', + 'info_usernet', + 'version', +] + +pytests_x86_64 = [ + 'cpu_queries', + 'mem_addr_space', + 'virtio_version', +] + +pytest = find_program('pytest', required: false) +if not pytest.found() + message('pytest not available ==> Disabled the qemu-pytests.') + subdir_done() +endif + +foreach dir : target_dirs + if not dir.endswith('-softmmu') + continue + endif + + target_base = dir.split('-')[0] + pytest_emulator = emulators['qemu-system-' + target_base] + target_pytests = get_variable('pytests_' + target_base, []) + pytests_generic + + test_deps = roms + pytest_env = environment() + if have_tools + pytest_env.set('PYTEST_QEMU_IMG', './qemu-img') + test_deps += [qemu_img] + endif + pytest_env.set('PYTEST_QEMU_BINARY', meson.global_build_root() / 'qemu-system-' + target_base) + pytest_env.set('PYTEST_SOURCE_ROOT', meson.project_source_root()) + pytest_env.set('PYTEST_BUILD_ROOT', meson.project_build_root()) + pytest_env.set('PYTHONPATH', meson.project_source_root() / 'python') + + foreach test : target_pytests + test('pytest-@0@/@1@'.format(target_base, test), + pytest, + depends: [test_deps, pytest_emulator, emulator_modules], + env: pytest_env, + args: [meson.current_source_dir() / 'test_' + test + '.py'], + timeout: slow_pytests.get(test, 60), + priority: slow_pytests.get(test, 60), + suite: ['pytest', 'pytest-' + target_base]) + endforeach +endforeach