| Message ID | 20220808105125.21356-10-stefan.herbrechtsmeier-oss@weidmueller.com |
|---|---|
| State | Superseded |
| Delegated to: | Simon Glass |
| Headers | show |
| Series | [v2,01/10] binman: Skip elf tests if python elftools is not available | expand |
Hi Stefan, On Mon, 8 Aug 2022 at 04:52, Stefan Herbrechtsmeier <stefan.herbrechtsmeier-oss@weidmueller.com> wrote: > > From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com> > > Add zstd bintool to binman to support on-the-fly compression. > > Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com> > > --- > > Changes in v2: > - Added > > tools/binman/btool/zstd.py | 30 ++++++++++++++++++++++++++++++ > tools/binman/comp_util.py | 18 +++++++++--------- > tools/binman/etype/blob_dtb.py | 4 ++++ > tools/binman/ftest.py | 3 ++- > 4 files changed, 45 insertions(+), 10 deletions(-) > create mode 100644 tools/binman/btool/zstd.py Reviewed-by: Simon Glass <sjg@chromium.org> But as you noted this does cause CI issues: ERROR: binman.ftest.TestFunctional.testCompressions (subunit.RemotedTestCase) binman.ftest.TestFunctional.testCompressions ---------------------------------------------------------------------- testtools.testresult.real._StringException: Traceback (most recent call last): TypeError: a bytes-like object is required, not 'NoneType' ====================================================================== FAIL: binman.ftest.TestFunctional.testVersions (subunit.RemotedTestCase) binman.ftest.TestFunctional.testVersions ---------------------------------------------------------------------- testtools.testresult.real._StringException: stdout: {{{ bzip2 - 1.0.8 gzip - 1.10 lz4 - v1.9.2 lzma - 9.22 beta lzo - v1.04 xz - 5.2.4 zstd - unknown }}} Traceback (most recent call last): AssertionError: Regex didn't match: '^v?[0-9]+[0-9.]*' not found in 'unknown' One option is to check if zstd is available, using is_present() in your testCompressions() function. Regards, Simon
Hi Simon, Am 13.08.2022 um 16:59 schrieb Simon Glass: > Hi Stefan, > > On Mon, 8 Aug 2022 at 04:52, Stefan Herbrechtsmeier > <stefan.herbrechtsmeier-oss@weidmueller.com> wrote: >> >> From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com> >> >> Add zstd bintool to binman to support on-the-fly compression. >> >> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com> >> >> --- >> >> Changes in v2: >> - Added >> >> tools/binman/btool/zstd.py | 30 ++++++++++++++++++++++++++++++ >> tools/binman/comp_util.py | 18 +++++++++--------- >> tools/binman/etype/blob_dtb.py | 4 ++++ >> tools/binman/ftest.py | 3 ++- >> 4 files changed, 45 insertions(+), 10 deletions(-) >> create mode 100644 tools/binman/btool/zstd.py > > Reviewed-by: Simon Glass <sjg@chromium.org> > > But as you noted this does cause CI issues: > > ERROR: binman.ftest.TestFunctional.testCompressions (subunit.RemotedTestCase) > binman.ftest.TestFunctional.testCompressions > ---------------------------------------------------------------------- > testtools.testresult.real._StringException: Traceback (most recent call last): > TypeError: a bytes-like object is required, not 'NoneType' > ====================================================================== > FAIL: binman.ftest.TestFunctional.testVersions (subunit.RemotedTestCase) > binman.ftest.TestFunctional.testVersions > ---------------------------------------------------------------------- > testtools.testresult.real._StringException: stdout: {{{ > bzip2 - 1.0.8 > gzip - 1.10 > lz4 - v1.9.2 > lzma - 9.22 beta > lzo - v1.04 > xz - 5.2.4 > zstd - unknown > }}} > Traceback (most recent call last): > AssertionError: Regex didn't match: '^v?[0-9]+[0-9.]*' not found in 'unknown' > > One option is to check if zstd is available, using is_present() in > your testCompressions() function. I have add a new function to check the present of the tools in each test. Regards Stefan
diff --git a/tools/binman/btool/zstd.py b/tools/binman/btool/zstd.py new file mode 100644 index 0000000000..299bd37126 --- /dev/null +++ b/tools/binman/btool/zstd.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG +# Stefan Herbrechtsmeier <stefan.herbrechtsmeier@weidmueller.com> +# +"""Bintool implementation for zstd + +zstd allows compression and decompression of files. + +Documentation is available via:: + + man zstd +""" + +from binman import bintool + +# pylint: disable=C0103 +class Bintoolzstd(bintool.BintoolPacker): + """Compression/decompression using the zstd algorithm + + This bintool supports running `zstd` to compress and decompress data, as + used by binman. + + It is also possible to fetch the tool, which uses `apt` to install it. + + Documentation is available via:: + + man zstd + """ + def __init__(self, name): + super().__init__(name) diff --git a/tools/binman/comp_util.py b/tools/binman/comp_util.py index 35835450e2..0f15fae600 100644 --- a/tools/binman/comp_util.py +++ b/tools/binman/comp_util.py @@ -11,7 +11,7 @@ from binman import bintool from patman import tools # Supported compressions -COMPRESSIONS = ['bzip2', 'gzip', 'lz4', 'lzma', 'lzo', 'xz'] +COMPRESSIONS = ['bzip2', 'gzip', 'lz4', 'lzma', 'lzo', 'xz', 'zstd'] bintools = {} @@ -34,14 +34,14 @@ def compress(indata, algo): Note that for lzma this uses an old version of the algorithm, not that provided by xz. - This requires 'bzip2', 'gzip', 'lz4', 'lzma_alone' 'lzop' and 'xz' tools. - It also requires an output directory to be previously set up, by calling - PrepareOutputDir(). + This requires 'bzip2', 'gzip', 'lz4', 'lzma_alone' 'lzop', 'xz' and 'zstd' + tools. It also requires an output directory to be previously set up, by + calling PrepareOutputDir(). Args: indata (bytes): Input data to compress algo (str): Algorithm to use ('none', 'bzip2', 'gzip', 'lz4', 'lzma', - 'lzo' or 'xz') + 'lzo', 'xz' or 'zstd') Returns: bytes: Compressed data @@ -62,14 +62,14 @@ def decompress(indata, algo): Note that for lzma this uses an old version of the algorithm, not that provided by xz. - This requires 'bzip2', 'gzip', 'lz4', 'lzma_alone', 'lzop' and 'xz' tools. - It also requires an output directory to be previously set up, by calling - PrepareOutputDir(). + This requires 'bzip2', 'gzip', 'lz4', 'lzma_alone', 'lzop', 'xz' and 'zstd' + tools. It also requires an output directory to be previously set up, by + calling PrepareOutputDir(). Args: indata (bytes): Input data to decompress algo (str): Algorithm to use ('none', 'bzip2', 'gzip', 'lz4', 'lzma', - 'lzo' or 'xz') + 'lzo', 'xz' or 'zstd') Returns: (bytes) Compressed data diff --git a/tools/binman/etype/blob_dtb.py b/tools/binman/etype/blob_dtb.py index 652b8abd8f..8d0b88d5b0 100644 --- a/tools/binman/etype/blob_dtb.py +++ b/tools/binman/etype/blob_dtb.py @@ -45,6 +45,10 @@ class Entry_blob_dtb(Entry_blob): def ProcessContents(self): """Re-read the DTB contents so that we get any calculated properties""" _, indata = state.GetFdtContents(self.GetFdtEtype()) + + if self.compress == 'zstd' and self.prepend != 'length': + self.Raise('The zstd compression requires a length header') + data = self.CompressData(indata) if self.prepend == 'length': hdr = struct.pack('<I', len(data)) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index c9b67c48d6..98a1c7c9db 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -5266,7 +5266,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap def testPadding(self): """Test padding of compression algorithms""" - for algo in comp_util.COMPRESSIONS: + # Skip zstd because it doesn't support padding + for algo in [a for a in comp_util.COMPRESSIONS if a != 'zstd']: data = comp_util.compress(COMPRESS_DATA, algo) data = data + bytes([0]) * 64 orig = comp_util.decompress(data, algo)