Message ID | f57167b1488b3c5ad76362dd8bca1b12d281aa19.1696522656.git.yann.morin.1998@free.fr |
---|---|
State | New |
Headers | show |
Series | support/runtime-test: extend graphic testing for weston + flutter (branch yem/flutter) | expand |
Hi Yann, Thanks for this patch! On 05/10/2023 18:17, Yann E. MORIN wrote: > In 4edb0e3456ef (support/testing/tests/package/test_weston.py: new > runtime test), the weston test was introduced, and thus was the first > that needed to test that rendering was happening. > > Now we also have a test for a flutter application, and we'll want to > have it test the rendering too. > > Move the corresponding code to a helper that can be reused by other > tests, rather than duplicate (or reinvent) it. > > Switch weston to using that new helper. > > Signed-off-by: Yann E. MORIN <yann.morin.1998@free.fr> > Cc: Julien Olivain <ju.o@free.fr> > --- > support/testing/tests/graphics_base.py | 39 +++++++++++++ > support/testing/tests/package/test_weston.py | 61 ++++---------------- > 2 files changed, 49 insertions(+), 51 deletions(-) > create mode 100644 support/testing/tests/graphics_base.py > > diff --git a/support/testing/tests/graphics_base.py > b/support/testing/tests/graphics_base.py > new file mode 100644 > index 0000000000..15a4c00bb2 > --- /dev/null > +++ b/support/testing/tests/graphics_base.py > @@ -0,0 +1,39 @@ > +class GraphicsBase: For those graphics tests (llvmpipe/swrast Mesa3D, VKMS) to work properly, there is some common mandatory configurations which will always be needed. I am mainly thinking about: BR2_PACKAGE_LIBDRM=y BR2_PACKAGE_MESA3D=y BR2_PACKAGE_MESA3D_GALLIUM_DRIVER_SWRAST=y BR2_PACKAGE_MESA3D_LLVM=y BR2_PACKAGE_MESA3D_OPENGL_EGL=y BR2_PACKAGE_MESA3D_OPENGL_ES=y The Kernel will also always need at least: CONFIG_DEBUG_FS=y CONFIG_DRM_VKMS=y In term of cpu architecture and emulated machine, there is no mandatory requirements, but to achieve decent test execution speed, a fast "machines" that also allow SMP and higher amount of memory are recommended. armv5-7 machines can be limited in maximum amount of memory and slow to execute Mesa3D. So x86_64 pc, Aarch64 or RISC-V 64bit virt will generally be architectures/machines of choice. Do you think it would be relevant to provide some base reference Buildroot+Kernel configs or templates, and the emulator startup command line in this base class? > + def get_n_fb_crc(self, *, count=10, uniq=False, timeout=-1): > + """ > + Return count DRM CRC from the framebuffer. If uniq is True, > + only unique CRCs are returned (which may be less than the > + requested cont). > + Returns a possibly empty list of integers. > + Set timeout to -1 for no timeout, or to a positive number for > + a timeout of that many seconds. > + """ > + # DRM CRCs are exposed through a sysfs pseudo file > + try: > + self.debugfs_mounted > + except AttributeError: > + # Note: some init system (e.g. systemd) may have this > already > + # mounted, so check beforehand > + self.assertRunOk("mountpoint /sys/kernel/debug/ || mount > -t debugfs none /sys/kernel/debug/") > + self.debugfs_mounted = True > + > + # The first column is the frame number, the second column is > the > + # CRC measure. We use "head" to get the needed CRC count. > + disp_crc_path = "/sys/kernel/debug/dri/0/crtc-0/crc/data" > + cmd = f"head -{count} {disp_crc_path}" > + > + # The DRM CRC sysfs pseudo file lines are terminated by '\n' > + # and '\0'. We remove the '\0' to have a text-only output. > + cmd += " | tr -d '\\000'" > + > + # Finally, we drop the frame counter, and keep only the second > + # column (CRC values) > + cmd += " | cut -f 2 -d ' '" > + > + if uniq: > + cmd += " | sort -u" > + > + output, exit_code = self.emulator.run(cmd, timeout=timeout) > + self.assertTrue(exit_code == 0, f"'{cmd}' failed with exit > code {exit_code}") > + > + return [int(crc, 16) for crc in output] > diff --git a/support/testing/tests/package/test_weston.py > b/support/testing/tests/package/test_weston.py > index df1b7a4135..f37a73565f 100644 > --- a/support/testing/tests/package/test_weston.py > +++ b/support/testing/tests/package/test_weston.py > @@ -2,9 +2,10 @@ import os > import time > > import infra.basetest > +from ..graphics_base import GraphicsBase > > > -class TestWeston(infra.basetest.BRTest): > +class TestWeston(infra.basetest.BRTest, GraphicsBase): > config = \ > """ > BR2_aarch64=y > @@ -36,31 +37,6 @@ class TestWeston(infra.basetest.BRTest): > > infra.filepath("tests/package/test_weston/linux-vkms.fragment") > ) > > - def gen_read_disp_crcs_cmd(self, count=1): > - # DRM CRCs are exposed through a sysfs pseudo file, one > measure > - # per line. The first column is the frame number, the second > - # column is the CRC measure. We use "head" to get the needed > - # CRC count. > - disp_crc_path = "/sys/kernel/debug/dri/0/crtc-0/crc/data" > - cmd = f"head -{count} {disp_crc_path}" > - > - # The DRM CRC sysfs pseudo file lines are terminated by '\n' > - # and '\0'. We remove the '\0' to have a text-only output. > - cmd += " | tr -d '\\000'" > - > - # Finally, we drop the frame counter, and keep only the second > - # column (CRC values) > - cmd += " | cut -f 2 -d ' '" > - > - return cmd > - > - def gen_count_unique_disp_crcs_cmd(self, count=10): > - # We get the command generating one CRC per line... > - cmd = self.gen_read_disp_crcs_cmd(count) > - # ...then count the number of unique values > - cmd += " | uniq | wc -l" > - return cmd > - > def start_weston(self): > self.assertRunOk("export XDG_RUNTIME_DIR=/tmp") > > @@ -106,25 +82,14 @@ class TestWeston(infra.basetest.BRTest): > # Check a simple info client can communicate with the > compositor > self.assertRunOk("wayland-info", timeout=10) > > - # This test will use the Kernel VKMS DRM Display CRC support, > - # which is exposed in debugfs. See: > - # > https://docs.kernel.org/gpu/drm-uapi.html#display-crc-support > - self.assertRunOk("mount -t debugfs none /sys/kernel/debug/") > - > # We get 10 consecutive DRM frame CRCs and count how many > # unique CRCs we have. Since weston is supposed to run idle, > # we should have 10 times the same display CRC. > - cmd = self.gen_count_unique_disp_crcs_cmd() > - output, exit_code = self.emulator.run(cmd) > - self.assertEqual(exit_code, 0) > - self.assertEqual(int(output[0]), 1) > + self.assertTrue(len(self.get_n_fb_crc(uniq=True)) == 1) > > # We save the CRC value of an empty weston desktop for > # later... > - cmd = self.gen_read_disp_crcs_cmd() > - output, exit_code = self.emulator.run(cmd) > - self.assertEqual(exit_code, 0) > - weston_desktop_crc = int(output[0], 16) > + weston_desktop_crc = self.get_n_fb_crc(count=1)[0] > > # We start the weston-simple-egl in background... Every > # rendered frame is supposed to be different (as the triangle > @@ -138,10 +103,8 @@ class TestWeston(infra.basetest.BRTest): > # display something, we are now supposed to measure a > # different display CRC than the one we measured when the > # desktop was empty. > - cmd = self.gen_read_disp_crcs_cmd() > - output, exit_code = self.emulator.run(cmd) > - self.assertEqual(exit_code, 0) > - self.assertNotEqual(int(output[0], 16), weston_desktop_crc) > + crc = self.get_n_fb_crc(count=1)[0] > + self.assertNotEqual(crc, weston_desktop_crc) > > # While weston-simple-egl is running, we check the VKMS DRM > # CRCs are now changing. We get many CRCs, one per display > @@ -152,10 +115,8 @@ class TestWeston(infra.basetest.BRTest): > # remain very permissive to slow emulation situations. > # Increase timeout, as the command is expected to run about > 5s, > # which is the default timeout. > - cmd = self.gen_count_unique_disp_crcs_cmd(300) > - output, exit_code = self.emulator.run(cmd, timeout=10) > - self.assertEqual(exit_code, 0) > - self.assertGreaterEqual(int(output[0]), 5) > + crcs = self.get_n_fb_crc(count=300, timeout=10) > + self.assertGreaterEqual(len(crcs), 5) > > # We stop weston-simple-egl, and sleep a bit to let Weston do > # its cleanup and desktop repaint refresh... > @@ -165,10 +126,8 @@ class TestWeston(infra.basetest.BRTest): > # After we stopped the application, we should have the initial > # weston desktop background. The CRC we measure now should be > # the same as the one we saved earlier. > - cmd = self.gen_read_disp_crcs_cmd() > - output, exit_code = self.emulator.run(cmd) > - self.assertEqual(exit_code, 0) > - self.assertEqual(int(output[0], 16), weston_desktop_crc) > + crc = self.get_n_fb_crc(count=1)[0] > + self.assertEqual(crc, weston_desktop_crc) > > self.stop_weston() > > -- > 2.25.1 Best regards, Julien.
diff --git a/support/testing/tests/graphics_base.py b/support/testing/tests/graphics_base.py new file mode 100644 index 0000000000..15a4c00bb2 --- /dev/null +++ b/support/testing/tests/graphics_base.py @@ -0,0 +1,39 @@ +class GraphicsBase: + def get_n_fb_crc(self, *, count=10, uniq=False, timeout=-1): + """ + Return count DRM CRC from the framebuffer. If uniq is True, + only unique CRCs are returned (which may be less than the + requested cont). + Returns a possibly empty list of integers. + Set timeout to -1 for no timeout, or to a positive number for + a timeout of that many seconds. + """ + # DRM CRCs are exposed through a sysfs pseudo file + try: + self.debugfs_mounted + except AttributeError: + # Note: some init system (e.g. systemd) may have this already + # mounted, so check beforehand + self.assertRunOk("mountpoint /sys/kernel/debug/ || mount -t debugfs none /sys/kernel/debug/") + self.debugfs_mounted = True + + # The first column is the frame number, the second column is the + # CRC measure. We use "head" to get the needed CRC count. + disp_crc_path = "/sys/kernel/debug/dri/0/crtc-0/crc/data" + cmd = f"head -{count} {disp_crc_path}" + + # The DRM CRC sysfs pseudo file lines are terminated by '\n' + # and '\0'. We remove the '\0' to have a text-only output. + cmd += " | tr -d '\\000'" + + # Finally, we drop the frame counter, and keep only the second + # column (CRC values) + cmd += " | cut -f 2 -d ' '" + + if uniq: + cmd += " | sort -u" + + output, exit_code = self.emulator.run(cmd, timeout=timeout) + self.assertTrue(exit_code == 0, f"'{cmd}' failed with exit code {exit_code}") + + return [int(crc, 16) for crc in output] diff --git a/support/testing/tests/package/test_weston.py b/support/testing/tests/package/test_weston.py index df1b7a4135..f37a73565f 100644 --- a/support/testing/tests/package/test_weston.py +++ b/support/testing/tests/package/test_weston.py @@ -2,9 +2,10 @@ import os import time import infra.basetest +from ..graphics_base import GraphicsBase -class TestWeston(infra.basetest.BRTest): +class TestWeston(infra.basetest.BRTest, GraphicsBase): config = \ """ BR2_aarch64=y @@ -36,31 +37,6 @@ class TestWeston(infra.basetest.BRTest): infra.filepath("tests/package/test_weston/linux-vkms.fragment") ) - def gen_read_disp_crcs_cmd(self, count=1): - # DRM CRCs are exposed through a sysfs pseudo file, one measure - # per line. The first column is the frame number, the second - # column is the CRC measure. We use "head" to get the needed - # CRC count. - disp_crc_path = "/sys/kernel/debug/dri/0/crtc-0/crc/data" - cmd = f"head -{count} {disp_crc_path}" - - # The DRM CRC sysfs pseudo file lines are terminated by '\n' - # and '\0'. We remove the '\0' to have a text-only output. - cmd += " | tr -d '\\000'" - - # Finally, we drop the frame counter, and keep only the second - # column (CRC values) - cmd += " | cut -f 2 -d ' '" - - return cmd - - def gen_count_unique_disp_crcs_cmd(self, count=10): - # We get the command generating one CRC per line... - cmd = self.gen_read_disp_crcs_cmd(count) - # ...then count the number of unique values - cmd += " | uniq | wc -l" - return cmd - def start_weston(self): self.assertRunOk("export XDG_RUNTIME_DIR=/tmp") @@ -106,25 +82,14 @@ class TestWeston(infra.basetest.BRTest): # Check a simple info client can communicate with the compositor self.assertRunOk("wayland-info", timeout=10) - # This test will use the Kernel VKMS DRM Display CRC support, - # which is exposed in debugfs. See: - # https://docs.kernel.org/gpu/drm-uapi.html#display-crc-support - self.assertRunOk("mount -t debugfs none /sys/kernel/debug/") - # We get 10 consecutive DRM frame CRCs and count how many # unique CRCs we have. Since weston is supposed to run idle, # we should have 10 times the same display CRC. - cmd = self.gen_count_unique_disp_crcs_cmd() - output, exit_code = self.emulator.run(cmd) - self.assertEqual(exit_code, 0) - self.assertEqual(int(output[0]), 1) + self.assertTrue(len(self.get_n_fb_crc(uniq=True)) == 1) # We save the CRC value of an empty weston desktop for # later... - cmd = self.gen_read_disp_crcs_cmd() - output, exit_code = self.emulator.run(cmd) - self.assertEqual(exit_code, 0) - weston_desktop_crc = int(output[0], 16) + weston_desktop_crc = self.get_n_fb_crc(count=1)[0] # We start the weston-simple-egl in background... Every # rendered frame is supposed to be different (as the triangle @@ -138,10 +103,8 @@ class TestWeston(infra.basetest.BRTest): # display something, we are now supposed to measure a # different display CRC than the one we measured when the # desktop was empty. - cmd = self.gen_read_disp_crcs_cmd() - output, exit_code = self.emulator.run(cmd) - self.assertEqual(exit_code, 0) - self.assertNotEqual(int(output[0], 16), weston_desktop_crc) + crc = self.get_n_fb_crc(count=1)[0] + self.assertNotEqual(crc, weston_desktop_crc) # While weston-simple-egl is running, we check the VKMS DRM # CRCs are now changing. We get many CRCs, one per display @@ -152,10 +115,8 @@ class TestWeston(infra.basetest.BRTest): # remain very permissive to slow emulation situations. # Increase timeout, as the command is expected to run about 5s, # which is the default timeout. - cmd = self.gen_count_unique_disp_crcs_cmd(300) - output, exit_code = self.emulator.run(cmd, timeout=10) - self.assertEqual(exit_code, 0) - self.assertGreaterEqual(int(output[0]), 5) + crcs = self.get_n_fb_crc(count=300, timeout=10) + self.assertGreaterEqual(len(crcs), 5) # We stop weston-simple-egl, and sleep a bit to let Weston do # its cleanup and desktop repaint refresh... @@ -165,10 +126,8 @@ class TestWeston(infra.basetest.BRTest): # After we stopped the application, we should have the initial # weston desktop background. The CRC we measure now should be # the same as the one we saved earlier. - cmd = self.gen_read_disp_crcs_cmd() - output, exit_code = self.emulator.run(cmd) - self.assertEqual(exit_code, 0) - self.assertEqual(int(output[0], 16), weston_desktop_crc) + crc = self.get_n_fb_crc(count=1)[0] + self.assertEqual(crc, weston_desktop_crc) self.stop_weston()
In 4edb0e3456ef (support/testing/tests/package/test_weston.py: new runtime test), the weston test was introduced, and thus was the first that needed to test that rendering was happening. Now we also have a test for a flutter application, and we'll want to have it test the rendering too. Move the corresponding code to a helper that can be reused by other tests, rather than duplicate (or reinvent) it. Switch weston to using that new helper. Signed-off-by: Yann E. MORIN <yann.morin.1998@free.fr> Cc: Julien Olivain <ju.o@free.fr> --- support/testing/tests/graphics_base.py | 39 +++++++++++++ support/testing/tests/package/test_weston.py | 61 ++++---------------- 2 files changed, 49 insertions(+), 51 deletions(-) create mode 100644 support/testing/tests/graphics_base.py