[PATCH 2/3] binman: Remove header from compressed data

[Simon Glass]

Hi Stefan,

On Tue, 2 Aug 2022 at 06:29, Stefan Herbrechtsmeier
<stefan.herbrechtsmeier-oss at weidmueller.com> wrote:
>
> From: Stefan Herbrechtsmeier <stefan.herbrechtsmeier at weidmueller.com>
>
> Remove header from compressed data because this is uncommon, not
> supported by U-Boot and incompatible with external compressed artifacts.
>
> The header was introduced as part of commit eb0f4a4cb402 ("binman:
> Support replacing data in a cbfs") to allow device tree entries to be
> larger that the compressed contents. Regarding the commit "this is
> necessary to cope with a compressed device tree being updated in such a
> way that it shrinks after the entry size is already set (an obscure
> case)". This case need to be fixed without influence the compressed data
> by itself.

I was not able to find a way around this due to the chicken-and egg
problem. Compressed data has an unpredictable size and adding an extra
uncompressed byte may increase or decrease the compressed size.

So my solution was to add the header.

It is optional though, so can we perhaps have a property in the
description to enable it?

Regards,
Simon


>
> Signed-off-by: Stefan Herbrechtsmeier <stefan.herbrechtsmeier at weidmueller.com>
>
> ---
>
>  tools/binman/cbfs_util.py |  8 ++++----
>  tools/binman/comp_util.py | 11 ++---------
>  tools/binman/entry.py     |  2 +-
>  3 files changed, 7 insertions(+), 14 deletions(-)
>
> diff --git a/tools/binman/cbfs_util.py b/tools/binman/cbfs_util.py
> index 9cad03886f..a1836f4ad3 100644
> --- a/tools/binman/cbfs_util.py
> +++ b/tools/binman/cbfs_util.py
> @@ -241,9 +241,9 @@ class CbfsFile(object):
>          """Handle decompressing data if necessary"""
>          indata = self.data
>          if self.compress == COMPRESS_LZ4:
> -            data = comp_util.decompress(indata, 'lz4', with_header=False)
> +            data = comp_util.decompress(indata, 'lz4')
>          elif self.compress == COMPRESS_LZMA:
> -            data = comp_util.decompress(indata, 'lzma', with_header=False)
> +            data = comp_util.decompress(indata, 'lzma')
>          else:
>              data = indata
>          self.memlen = len(data)
> @@ -362,9 +362,9 @@ class CbfsFile(object):
>          elif self.ftype == TYPE_RAW:
>              orig_data = data
>              if self.compress == COMPRESS_LZ4:
> -                data = comp_util.compress(orig_data, 'lz4', with_header=False)
> +                data = comp_util.compress(orig_data, 'lz4')
>              elif self.compress == COMPRESS_LZMA:
> -                data = comp_util.compress(orig_data, 'lzma', with_header=False)
> +                data = comp_util.compress(orig_data, 'lzma')
>              self.memlen = len(orig_data)
>              self.data_len = len(data)
>              attr = struct.pack(ATTR_COMPRESSION_FORMAT,
> diff --git a/tools/binman/comp_util.py b/tools/binman/comp_util.py
> index dc76adab35..269bbf7975 100644
> --- a/tools/binman/comp_util.py
> +++ b/tools/binman/comp_util.py
> @@ -3,7 +3,6 @@
>  #
>  """Utilities to compress and decompress data"""
>
> -import struct
>  import tempfile
>
>  from binman import bintool
> @@ -16,7 +15,7 @@ LZMA_ALONE = bintool.Bintool.create('lzma_alone')
>  HAVE_LZMA_ALONE = LZMA_ALONE.is_present()
>
>
> -def compress(indata, algo, with_header=True):
> +def compress(indata, algo):
>      """Compress some data using a given algorithm
>
>      Note that for lzma this uses an old version of the algorithm, not that
> @@ -41,12 +40,9 @@ def compress(indata, algo, with_header=True):
>          data = LZMA_ALONE.compress(indata)
>      else:
>          raise ValueError("Unknown algorithm '%s'" % algo)
> -    if with_header:
> -        hdr = struct.pack('<I', len(data))
> -        data = hdr + data
>      return data
>
> -def decompress(indata, algo, with_header=True):
> +def decompress(indata, algo):
>      """Decompress some data using a given algorithm
>
>      Note that for lzma this uses an old version of the algorithm, not that
> @@ -64,9 +60,6 @@ def decompress(indata, algo, with_header=True):
>      """
>      if algo == 'none':
>          return indata
> -    if with_header:
> -        data_len = struct.unpack('<I', indata[:4])[0]
> -        indata = indata[4:4 + data_len]
>      if algo == 'lz4':
>          data = LZ4.decompress(indata)
>      elif algo == 'lzma':
> diff --git a/tools/binman/entry.py b/tools/binman/entry.py
> index a07a588864..8cbfd43af9 100644
> --- a/tools/binman/entry.py
> +++ b/tools/binman/entry.py
> @@ -1069,7 +1069,7 @@ features to produce new behaviours.
>              indata: Data to compress
>
>          Returns:
> -            Compressed data (first word is the compressed size)
> +            Compressed data
>          """
>          self.uncomp_data = indata
>          if self.compress != 'none':
> --
> 2.30.2
>