| Message ID | 20240318154621.2361161-14-npiggin@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | replay: fixes and new test cases | expand |
Nicholas Piggin <npiggin@gmail.com> writes: > replay_linux tests with virtio on aarch64 gciv3 and x86-64 q35 machines > seems to be more reliable now, so timeouts are no longer expected. > pc_i440fx, gciv2, and non-virtio still have problems, so mark them as > flaky: they are not just long-running, but can hang indefinitely. > > These tests take about 400 seconds each, so add the SPEED=slow guard. > > Signed-off-by: Nicholas Piggin <npiggin@gmail.com> > --- > tests/avocado/replay_linux.py | 10 +++++++--- > 1 file changed, 7 insertions(+), 3 deletions(-) > > diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py > index b3b91ddd9a..b3b74a367c 100644 > --- a/tests/avocado/replay_linux.py > +++ b/tests/avocado/replay_linux.py > @@ -118,7 +118,7 @@ def run_replay_dump(self, replay_path): > except subprocess.CalledProcessError: > self.fail('replay-dump.py failed') > > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') > class ReplayLinuxX8664(ReplayLinux): > """ > :avocado: tags=arch:x86_64 > @@ -127,19 +127,21 @@ class ReplayLinuxX8664(ReplayLinux): > > chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0' > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') > def test_pc_i440fx(self): > """ > :avocado: tags=machine:pc > """ > self.run_rr(shift=1) > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') > def test_pc_q35(self): > """ > :avocado: tags=machine:q35 > """ > self.run_rr(shift=3) > > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') > class ReplayLinuxX8664Virtio(ReplayLinux): > """ > :avocado: tags=arch:x86_64 > @@ -153,6 +155,7 @@ class ReplayLinuxX8664Virtio(ReplayLinux): > > chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0' > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') > def test_pc_i440fx(self): > """ > :avocado: tags=machine:pc > @@ -165,7 +168,7 @@ def test_pc_q35(self): > """ > self.run_rr(shift=3) > > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') > class ReplayLinuxAarch64(ReplayLinux): > """ > :avocado: tags=accel:tcg > @@ -187,6 +190,7 @@ def get_common_args(self): > '-device', 'virtio-rng-pci,rng=rng0', > '-object', 'rng-builtin,id=rng0') > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is > unstable') This needs to apply to both I think: (5/7) ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv2: SKIP: Test is unstable (6/7) ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3: INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred: Timeout reached\nOriginal status: ERROR\n{'name': '6-./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3', 'logdir': '/home/alex/avocado/job-results/job-2024-03-19T16.50-686495d/test-results/... (1800.17 s) With that: Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
On Wed Mar 20, 2024 at 3:57 AM AEST, Alex Bennée wrote: > Nicholas Piggin <npiggin@gmail.com> writes: > > > replay_linux tests with virtio on aarch64 gciv3 and x86-64 q35 machines > > seems to be more reliable now, so timeouts are no longer expected. > > pc_i440fx, gciv2, and non-virtio still have problems, so mark them as > > flaky: they are not just long-running, but can hang indefinitely. > > > > These tests take about 400 seconds each, so add the SPEED=slow guard. > > > > Signed-off-by: Nicholas Piggin <npiggin@gmail.com> > > --- > > tests/avocado/replay_linux.py | 10 +++++++--- > > 1 file changed, 7 insertions(+), 3 deletions(-) > > > > diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py > > index b3b91ddd9a..b3b74a367c 100644 > > --- a/tests/avocado/replay_linux.py > > +++ b/tests/avocado/replay_linux.py > > @@ -118,7 +118,7 @@ def run_replay_dump(self, replay_path): > > except subprocess.CalledProcessError: > > self.fail('replay-dump.py failed') > > > > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') > > class ReplayLinuxX8664(ReplayLinux): > > """ > > :avocado: tags=arch:x86_64 > > @@ -127,19 +127,21 @@ class ReplayLinuxX8664(ReplayLinux): > > > > chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0' > > > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') > > def test_pc_i440fx(self): > > """ > > :avocado: tags=machine:pc > > """ > > self.run_rr(shift=1) > > > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') > > def test_pc_q35(self): > > """ > > :avocado: tags=machine:q35 > > """ > > self.run_rr(shift=3) > > > > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') > > class ReplayLinuxX8664Virtio(ReplayLinux): > > """ > > :avocado: tags=arch:x86_64 > > @@ -153,6 +155,7 @@ class ReplayLinuxX8664Virtio(ReplayLinux): > > > > chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0' > > > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') > > def test_pc_i440fx(self): > > """ > > :avocado: tags=machine:pc > > @@ -165,7 +168,7 @@ def test_pc_q35(self): > > """ > > self.run_rr(shift=3) > > > > -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') > > +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') > > class ReplayLinuxAarch64(ReplayLinux): > > """ > > :avocado: tags=accel:tcg > > @@ -187,6 +190,7 @@ def get_common_args(self): > > '-device', 'virtio-rng-pci,rng=rng0', > > '-object', 'rng-builtin,id=rng0') > > > > + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is > > unstable') > > This needs to apply to both I think: > > (5/7) ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv2: SKIP: Test is unstable > (6/7) > ./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3: > INTERRUPTED: Test interrupted by SIGTERM\nRunner error occurred: > Timeout reached\nOriginal status: ERROR\n{'name': > '6-./tests/avocado/replay_linux.py:ReplayLinuxAarch64.test_virt_gicv3', > 'logdir': > '/home/alex/avocado/job-results/job-2024-03-19T16.50-686495d/test-results/... > (1800.17 s) Oh, aarch64 is hanging for you too? It was passing for me... But I'll guard it out for now and maybe we can get back to it later. Thanks, Nick > > With that: > > Reviewed-by: Alex Bennée <alex.bennee@linaro.org>
diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py index b3b91ddd9a..b3b74a367c 100644 --- a/tests/avocado/replay_linux.py +++ b/tests/avocado/replay_linux.py @@ -118,7 +118,7 @@ def run_replay_dump(self, replay_path): except subprocess.CalledProcessError: self.fail('replay-dump.py failed') -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') class ReplayLinuxX8664(ReplayLinux): """ :avocado: tags=arch:x86_64 @@ -127,19 +127,21 @@ class ReplayLinuxX8664(ReplayLinux): chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0' + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') def test_pc_i440fx(self): """ :avocado: tags=machine:pc """ self.run_rr(shift=1) + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') def test_pc_q35(self): """ :avocado: tags=machine:q35 """ self.run_rr(shift=3) -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') class ReplayLinuxX8664Virtio(ReplayLinux): """ :avocado: tags=arch:x86_64 @@ -153,6 +155,7 @@ class ReplayLinuxX8664Virtio(ReplayLinux): chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0' + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') def test_pc_i440fx(self): """ :avocado: tags=machine:pc @@ -165,7 +168,7 @@ def test_pc_q35(self): """ self.run_rr(shift=3) -@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') +@skipUnless(os.getenv('SPEED') == 'slow', 'runtime limited') class ReplayLinuxAarch64(ReplayLinux): """ :avocado: tags=accel:tcg @@ -187,6 +190,7 @@ def get_common_args(self): '-device', 'virtio-rng-pci,rng=rng0', '-object', 'rng-builtin,id=rng0') + @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable') def test_virt_gicv2(self): """ :avocado: tags=machine:gic-version=2
replay_linux tests with virtio on aarch64 gciv3 and x86-64 q35 machines seems to be more reliable now, so timeouts are no longer expected. pc_i440fx, gciv2, and non-virtio still have problems, so mark them as flaky: they are not just long-running, but can hang indefinitely. These tests take about 400 seconds each, so add the SPEED=slow guard. Signed-off-by: Nicholas Piggin <npiggin@gmail.com> --- tests/avocado/replay_linux.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-)