[U-Boot] [PATCH 21/31] binman: Support compressed entries

Simon Glass sjg at chromium.org
Fri Sep 14 10:57:26 UTC 2018


Add support for compressing blob entries. This can help reduce image sizes
for many types of data. It requires that the firmware be able to
decompress the data at run-time.

Signed-off-by: Simon Glass <sjg at chromium.org>
---

 tools/binman/README               | 16 ++++++++++
 tools/binman/README.entries       |  7 +++++
 tools/binman/etype/blob.py        | 49 +++++++++++++++++++++++++------
 tools/binman/ftest.py             | 32 ++++++++++++++++++++
 tools/binman/test/83_compress.dts | 11 +++++++
 5 files changed, 106 insertions(+), 9 deletions(-)
 create mode 100644 tools/binman/test/83_compress.dts

diff --git a/tools/binman/README b/tools/binman/README
index 3dc8b9b80a8..34b83110f64 100644
--- a/tools/binman/README
+++ b/tools/binman/README
@@ -593,6 +593,22 @@ the device tree. These can be used by U-Boot at run-time to find the location
 of each entry.
 
 
+Compression
+-----------
+
+Binman support compression for 'blob' entries (those of type 'blob' and
+derivatives). To enable this for an entry, add a 'compression' property:
+
+    blob {
+        filename = "datafile";
+        compression = "lz4";
+    };
+
+The entry will then contain the compressed data, using the 'lz4' compression
+algorithm. Currently this is the only one that is supported.
+
+
+
 Map files
 ---------
 
diff --git a/tools/binman/README.entries b/tools/binman/README.entries
index 091fb5ce2b6..2cf7dc0338d 100644
--- a/tools/binman/README.entries
+++ b/tools/binman/README.entries
@@ -19,11 +19,18 @@ class by other entry types.
 
 Properties / Entry arguments:
     - filename: Filename of file to read into entry
+    - compress: Compression algorithm to use:
+        none: No compression
+        lz4: Use lz4 compression (via 'lz4' command-line utility)
 
 This entry reads data from a file and places it in the entry. The
 default filename is often specified specified by the subclass. See for
 example the 'u_boot' entry which provides the filename 'u-boot.bin'.
 
+If compression is enabled, an extra 'uncomp-size' property is written to
+the node (if enabled with -u) which provides the uncompressed size of the
+data.
+
 
 
 Entry: blob-dtb: A blob that holds a device tree
diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py
index 3f46eecf308..642a0e482a7 100644
--- a/tools/binman/etype/blob.py
+++ b/tools/binman/etype/blob.py
@@ -7,6 +7,7 @@
 
 from entry import Entry
 import fdt_util
+import state
 import tools
 
 class Entry_blob(Entry):
@@ -17,14 +18,23 @@ class Entry_blob(Entry):
 
     Properties / Entry arguments:
         - filename: Filename of file to read into entry
+        - compress: Compression algorithm to use:
+            none: No compression
+            lz4: Use lz4 compression (via 'lz4' command-line utility)
 
     This entry reads data from a file and places it in the entry. The
     default filename is often specified specified by the subclass. See for
     example the 'u_boot' entry which provides the filename 'u-boot.bin'.
+
+    If compression is enabled, an extra 'uncomp-size' property is written to
+    the node (if enabled with -u) which provides the uncompressed size of the
+    data.
     """
     def __init__(self, section, etype, node):
         Entry.__init__(self, section, etype, node)
-        self._filename = fdt_util.GetString(self._node, "filename", self.etype)
+        self._filename = fdt_util.GetString(self._node, 'filename', self.etype)
+        self._compress = fdt_util.GetString(self._node, 'compress', 'none')
+        self._uncompressed_size = None
 
     def ObtainContents(self):
         self._filename = self.GetDefaultFilename()
@@ -33,15 +43,36 @@ class Entry_blob(Entry):
         return True
 
     def ReadBlobContents(self):
-        with open(self._pathname) as fd:
-            # We assume the data is small enough to fit into memory. If this
-            # is used for large filesystem image that might not be true.
-            # In that case, Image.BuildImage() could be adjusted to use a
-            # new Entry method which can read in chunks. Then we could copy
-            # the data in chunks and avoid reading it all at once. For now
-            # this seems like an unnecessary complication.
-            self.SetContents(fd.read())
+        # We assume the data is small enough to fit into memory. If this
+        # is used for large filesystem image that might not be true.
+        # In that case, Image.BuildImage() could be adjusted to use a
+        # new Entry method which can read in chunks. Then we could copy
+        # the data in chunks and avoid reading it all at once. For now
+        # this seems like an unnecessary complication.
+        data = tools.ReadFile(self._pathname)
+        if self._compress == 'lz4':
+            self._uncompressed_size = len(data)
+            '''
+            import lz4  # Import this only if needed (python-lz4 dependency)
+
+            try:
+                data = lz4.frame.compress(data)
+            except AttributeError:
+                data = lz4.compress(data)
+            '''
+            data = tools.Run('lz4', '-c', self._pathname, )
+        self.SetContents(data)
         return True
 
     def GetDefaultFilename(self):
         return self._filename
+
+    def AddMissingProperties(self):
+        Entry.AddMissingProperties(self)
+        if self._compress != 'none':
+            state.AddZeroProp(self._node, 'uncomp-size')
+
+    def SetCalculatedProperties(self):
+        Entry.SetCalculatedProperties(self)
+        if self._uncompressed_size is not None:
+            state.SetInt(self._node, 'uncomp-size', self._uncompressed_size)
diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py
index 6bfef7b63a6..c5e7236c424 100644
--- a/tools/binman/ftest.py
+++ b/tools/binman/ftest.py
@@ -6,6 +6,7 @@
 #
 #    python -m unittest func_test.TestFunctional.testHelp
 
+import lz4
 from optparse import OptionParser
 import os
 import shutil
@@ -54,6 +55,7 @@ CROS_EC_RW_DATA       = 'ecrw'
 GBB_DATA              = 'gbbd'
 BMPBLK_DATA           = 'bmp'
 VBLOCK_DATA           = 'vblk'
+COMPRESS_DATA         = 'data to compress'
 
 
 class TestFunctional(unittest.TestCase):
@@ -116,6 +118,8 @@ class TestFunctional(unittest.TestCase):
         with open(self.TestFile('descriptor.bin')) as fd:
             TestFunctional._MakeInputFile('descriptor.bin', fd.read())
 
+        TestFunctional._MakeInputFile('compress', COMPRESS_DATA)
+
     @classmethod
     def tearDownClass(self):
         """Remove the temporary input directory and its contents"""
@@ -1505,6 +1509,34 @@ class TestFunctional(unittest.TestCase):
         finally:
             self._ResetDtbs()
 
+    def _decompress(self, data):
+        out = os.path.join(self._indir, 'lz4.tmp')
+        with open(out, 'wb') as fd:
+            fd.write(data)
+        return tools.Run('lz4', '-dc', out)
+        '''
+        try:
+            orig = lz4.frame.decompress(data)
+        except AttributeError:
+            orig = lz4.decompress(data)
+        '''
+
+    def testCompress(self):
+        """Test compression of blobs"""
+        data, _, _, out_dtb_fname = self._DoReadFileDtb('83_compress.dts',
+                                            use_real_dtb=True, update_dtb=True)
+        dtb = fdt.Fdt(out_dtb_fname)
+        dtb.Scan()
+        props = self._GetPropTree(dtb, ['size', 'uncomp-size'])
+        orig = self._decompress(data)
+        self.assertEquals(COMPRESS_DATA, orig)
+        expected = {
+            'blob:uncomp-size': len(COMPRESS_DATA),
+            'blob:size': len(data),
+            'size': len(data),
+            }
+        self.assertEqual(expected, props)
+
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/tools/binman/test/83_compress.dts b/tools/binman/test/83_compress.dts
new file mode 100644
index 00000000000..07813bdeaa3
--- /dev/null
+++ b/tools/binman/test/83_compress.dts
@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-2.0+
+/dts-v1/;
+
+/ {
+	binman {
+		blob {
+			filename = "compress";
+			compress = "lz4";
+		};
+	};
+};
-- 
2.19.0.397.gdd90340f6a-goog



More information about the U-Boot mailing list