[PATCH 3/7] binman: Allow extracting a file in an alternative format
Simon Glass
sjg at chromium.org
Wed Nov 24 05:09:50 CET 2021
In some cases entries encapsulate other data and it is useful to access
the data within. An example is the fdtmap which consists of a 16-byte
header, followed by a devicetree.
Provide an option to specify an alternative format when extracting files.
In the case of fdtmap, this is 'fdt', which produces an FDT file which can
be viewed with fdtdump.
Signed-off-by: Simon Glass <sjg at chromium.org>
---
tools/binman/binman.rst | 29 ++++++++++++++++
tools/binman/cmdline.py | 2 ++
tools/binman/control.py | 28 +++++++++++++---
tools/binman/entries.rst | 11 +++++-
tools/binman/entry.py | 37 +++++++++++++++++----
tools/binman/etype/atf_fip.py | 16 ++++++++-
tools/binman/etype/cbfs.py | 8 ++---
tools/binman/etype/fdtmap.py | 12 +++++++
tools/binman/etype/section.py | 23 ++++++++++---
tools/binman/ftest.py | 34 +++++++++++++++++++
tools/binman/image.py | 2 +-
tools/binman/test/213_fdtmap_alt_format.dts | 15 +++++++++
tools/binman/test/214_no_alt_format.dts | 13 ++++++++
13 files changed, 208 insertions(+), 22 deletions(-)
create mode 100644 tools/binman/test/213_fdtmap_alt_format.dts
create mode 100644 tools/binman/test/214_no_alt_format.dts
diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst
index 26f462ae16f..10389a52c4b 100644
--- a/tools/binman/binman.rst
+++ b/tools/binman/binman.rst
@@ -942,6 +942,35 @@ or just a selection::
$ binman extract -i image.bin "*u-boot*" -O outdir
+Some entry types have alternative formats, for example fdtmap which allows
+extracted just the devicetree binary without the fdtmap header::
+
+ $ binman extract -i /tmp/b/odroid-c4/image.bin -f out.dtb -F fdt fdtmap
+ $ fdtdump out.dtb
+ /dts-v1/;
+ // magic: 0xd00dfeed
+ // totalsize: 0x8ab (2219)
+ // off_dt_struct: 0x38
+ // off_dt_strings: 0x82c
+ // off_mem_rsvmap: 0x28
+ // version: 17
+ // last_comp_version: 2
+ // boot_cpuid_phys: 0x0
+ // size_dt_strings: 0x7f
+ // size_dt_struct: 0x7f4
+
+ / {
+ image-node = "binman";
+ image-pos = <0x00000000>;
+ size = <0x0011162b>;
+ ...
+
+Use `-F list` to see what alternative formats are available::
+
+ $ binman extract -i /tmp/b/odroid-c4/image.bin -F list
+ Flag (-F) Entry type Description
+ fdt fdtmap Extract the devicetree blob from the fdtmap
+
Replacing files in an image
---------------------------
diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py
index 2229316f10e..adc17547ae6 100644
--- a/tools/binman/cmdline.py
+++ b/tools/binman/cmdline.py
@@ -17,6 +17,8 @@ def make_extract_parser(subparsers):
"""
extract_parser = subparsers.add_parser('extract',
help='Extract files from an image')
+ extract_parser.add_argument('-F', '--format', type=str,
+ help='Select an alternative format for extracted data')
extract_parser.add_argument('-i', '--image', type=str, required=True,
help='Image filename to extract')
extract_parser.add_argument('-f', '--filename', type=str,
diff --git a/tools/binman/control.py b/tools/binman/control.py
index 304fc70f56f..7288a2639d5 100644
--- a/tools/binman/control.py
+++ b/tools/binman/control.py
@@ -200,8 +200,24 @@ def ReadEntry(image_fname, entry_path, decomp=True):
return entry.ReadData(decomp)
+def ShowAltFormats(image):
+ """Show alternative formats available for entries in the image
+
+ This shows a list of formats available.
+
+ Args:
+ image (Image): Image to check
+ """
+ alt_formats = {}
+ image.CheckAltFormats(alt_formats)
+ print('%-10s %-20s %s' % ('Flag (-F)', 'Entry type', 'Description'))
+ for name, val in alt_formats.items():
+ entry, helptext = val
+ print('%-10s %-20s %s' % (name, entry.etype, helptext))
+
+
def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
- decomp=True):
+ decomp=True, alt_format=None):
"""Extract the data from one or more entries and write it to files
Args:
@@ -217,6 +233,10 @@ def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
"""
image = Image.FromFile(image_fname)
+ if alt_format == 'list':
+ ShowAltFormats(image)
+ return
+
# Output an entry to a single file, as a special case
if output_fname:
if not entry_paths:
@@ -224,7 +244,7 @@ def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
if len(entry_paths) != 1:
raise ValueError('Must specify exactly one entry path to write with -f')
entry = image.FindEntryPath(entry_paths[0])
- data = entry.ReadData(decomp)
+ data = entry.ReadData(decomp, alt_format)
tools.WriteFile(output_fname, data)
tout.Notice("Wrote %#x bytes to file '%s'" % (len(data), output_fname))
return
@@ -236,7 +256,7 @@ def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
tout.Notice('%d entries match and will be written' % len(einfos))
for einfo in einfos:
entry = einfo.entry
- data = entry.ReadData(decomp)
+ data = entry.ReadData(decomp, alt_format)
path = entry.GetPath()[1:]
fname = os.path.join(outdir, path)
@@ -583,7 +603,7 @@ def Binman(args):
if args.cmd == 'extract':
ExtractEntries(args.image, args.filename, args.outdir, args.paths,
- not args.uncompressed)
+ not args.uncompressed, args.format)
if args.cmd == 'replace':
ReplaceEntries(args.image, args.filename, args.indir, args.paths,
diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst
index 84f828a6352..cf3a6e69599 100644
--- a/tools/binman/entries.rst
+++ b/tools/binman/entries.rst
@@ -468,6 +468,10 @@ Example output for a simple image with U-Boot and an FDT map::
If allow-repack is used then 'orig-offset' and 'orig-size' properties are
added as necessary. See the binman README.
+When extracting files, an alternative 'fdt' format is available for fdtmaps.
+Use `binman extract -F fdt ...` to use this. It will export a devicetree,
+without the fdtmap header, so it can be viewed with `fdtdump`.
+
Entry: files: A set of files arranged in a section
@@ -1009,7 +1013,7 @@ SetImagePos(image_pos):
Binman calls this after the image has been packed, to update the
location that all the entries ended up at.
-ReadChildData(child, decomp):
+ReadChildData(child, decomp, alt_format):
The default version of this may be good enough, if you are able to
implement SetImagePos() correctly. But that is a bit of a bypass, so
you can override this method to read from your custom file format. It
@@ -1022,6 +1026,11 @@ ReadChildData(child, decomp):
uncompress it first, then return the uncompressed data (`decomp` is
True). This is used by the `binman extract -U` option.
+ If your entry supports alternative formats, the alt_format provides the
+ alternative format that the user has selected. Your function should
+ return data in that format. This is used by the 'binman extract -l'
+ option.
+
Binman calls this when reading in an image, in order to populate all the
entries with the data from that image (`binman ls`).
diff --git a/tools/binman/entry.py b/tools/binman/entry.py
index e7a8365fd51..61642bf5017 100644
--- a/tools/binman/entry.py
+++ b/tools/binman/entry.py
@@ -815,7 +815,7 @@ features to produce new behaviours.
self.AddEntryInfo(entries, indent, self.name, self.etype, self.size,
self.image_pos, self.uncomp_size, self.offset, self)
- def ReadData(self, decomp=True):
+ def ReadData(self, decomp=True, alt_format=None):
"""Read the data for an entry from the image
This is used when the image has been read in and we want to extract the
@@ -832,19 +832,20 @@ features to produce new behaviours.
# although compressed sections are currently not supported
tout.Debug("ReadChildData section '%s', entry '%s'" %
(self.section.GetPath(), self.GetPath()))
- data = self.section.ReadChildData(self, decomp)
+ data = self.section.ReadChildData(self, decomp, alt_format)
return data
- def ReadChildData(self, child, decomp=True):
+ def ReadChildData(self, child, decomp=True, alt_format=None):
"""Read the data for a particular child entry
This reads data from the parent and extracts the piece that relates to
the given child.
Args:
- child: Child entry to read data for (must be valid)
- decomp: True to decompress any compressed data before returning it;
- False to return the raw, uncompressed data
+ child (Entry): Child entry to read data for (must be valid)
+ decomp (bool): True to decompress any compressed data before
+ returning it; False to return the raw, uncompressed data
+ alt_format (str): Alternative format to read in, or None
Returns:
Data for the child (bytes)
@@ -857,6 +858,20 @@ features to produce new behaviours.
self.ProcessContentsUpdate(data)
self.Detail('Loaded data size %x' % len(data))
+ def GetAltFormat(self, data, alt_format):
+ """Read the data for an extry in an alternative format
+
+ Supported formats are list in the documentation for each entry. An
+ example is fdtmap which provides .
+
+ Args:
+ data (bytes): Data to convert (this should have been produced by the
+ entry)
+ alt_format (str): Format to use
+
+ """
+ pass
+
def GetImage(self):
"""Get the image containing this entry
@@ -997,3 +1012,13 @@ features to produce new behaviours.
tout.Info("Node '%s': etype '%s': %s selected" %
(node.path, etype, new_etype))
return True
+
+ def CheckAltFormats(self, alt_formats):
+ """Add any alternative formats supported by this entry type
+
+ Args:
+ alt_formats (dict): Dict to add alt_formats to:
+ key: Name of alt format
+ value: Help text
+ """
+ pass
diff --git a/tools/binman/etype/atf_fip.py b/tools/binman/etype/atf_fip.py
index f9039e14c0a..fad2dd0e0eb 100644
--- a/tools/binman/etype/atf_fip.py
+++ b/tools/binman/etype/atf_fip.py
@@ -250,7 +250,21 @@ class Entry_atf_fip(Entry_section):
entry.offset = fent.offset
entry.image_pos = self.image_pos + entry.offset
- def ReadChildData(self, child, decomp=True):
+ def ReadChildData(self, child, decomp=True, alt_format=None):
+ """Read the data for a particular child entry
+
+ This reads data from the parent and extracts the piece that relates to
+ the given child.
+
+ Args:
+ child (Entry): Child entry to read data for (must be valid)
+ decomp (bool): True to decompress any compressed data before
+ returning it; False to return the raw, uncompressed data
+ alt_format (str): Alternative format to read in, or None
+
+ Returns:
+ bytes: Data for the child (bytes)
+ """
if not self.reader:
self.fip_data = super().ReadData(True)
self.reader = FipReader(self.fip_data)
diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py
index 2459388f842..cc1fbdf4b57 100644
--- a/tools/binman/etype/cbfs.py
+++ b/tools/binman/etype/cbfs.py
@@ -276,13 +276,13 @@ class Entry_cbfs(Entry):
def GetEntries(self):
return self._entries
- def ReadData(self, decomp=True):
- data = super().ReadData(True)
+ def ReadData(self, decomp=True, alt_format=None):
+ data = super().ReadData(True, alt_format)
return data
- def ReadChildData(self, child, decomp=True):
+ def ReadChildData(self, child, decomp=True, alt_format=None):
if not self.reader:
- data = super().ReadData(True)
+ data = super().ReadData(True, alt_format)
self.reader = cbfs_util.CbfsReader(data)
reader = self.reader
cfile = reader.files.get(child.name)
diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py
index 2339feeba8d..aaaf2de4383 100644
--- a/tools/binman/etype/fdtmap.py
+++ b/tools/binman/etype/fdtmap.py
@@ -74,6 +74,10 @@ class Entry_fdtmap(Entry):
If allow-repack is used then 'orig-offset' and 'orig-size' properties are
added as necessary. See the binman README.
+
+ When extracting files, an alternative 'fdt' format is available for fdtmaps.
+ Use `binman extract -F fdt ...` to use this. It will export a devicetree,
+ without the fdtmap header, so it can be viewed with `fdtdump`.
"""
def __init__(self, section, etype, node):
# Put these here to allow entry-docs and help to work without libfdt
@@ -86,6 +90,10 @@ class Entry_fdtmap(Entry):
from dtoc.fdt import Fdt
super().__init__(section, etype, node)
+ self.alt_formats = ['fdt']
+
+ def CheckAltFormats(self, alt_formats):
+ alt_formats['fdt'] = self, 'Extract the devicetree blob from the fdtmap'
def _GetFdtmap(self):
"""Build an FDT map from the entries in the current image
@@ -147,3 +155,7 @@ class Entry_fdtmap(Entry):
processing, e.g. the image-pos properties.
"""
return self.ProcessContentsUpdate(self._GetFdtmap())
+
+ def GetAltFormat(self, data, alt_format):
+ if alt_format == 'fdt':
+ return data[FDTMAP_HDR_LEN:]
diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py
index 6ce07dd37d7..43436a11f27 100644
--- a/tools/binman/etype/section.py
+++ b/tools/binman/etype/section.py
@@ -80,7 +80,7 @@ class Entry_section(Entry):
Binman calls this after the image has been packed, to update the
location that all the entries ended up at.
- ReadChildData(child, decomp):
+ ReadChildData(child, decomp, alt_format):
The default version of this may be good enough, if you are able to
implement SetImagePos() correctly. But that is a bit of a bypass, so
you can override this method to read from your custom file format. It
@@ -93,6 +93,11 @@ class Entry_section(Entry):
uncompress it first, then return the uncompressed data (`decomp` is
True). This is used by the `binman extract -U` option.
+ If your entry supports alternative formats, the alt_format provides the
+ alternative format that the user has selected. Your function should
+ return data in that format. This is used by the 'binman extract -l'
+ option.
+
Binman calls this when reading in an image, in order to populate all the
entries with the data from that image (`binman ls`).
@@ -750,9 +755,9 @@ class Entry_section(Entry):
"""
return self._sort
- def ReadData(self, decomp=True):
+ def ReadData(self, decomp=True, alt_format=None):
tout.Info("ReadData path='%s'" % self.GetPath())
- parent_data = self.section.ReadData(True)
+ parent_data = self.section.ReadData(True, alt_format)
offset = self.offset - self.section._skip_at_start
data = parent_data[offset:offset + self.size]
tout.Info(
@@ -761,9 +766,9 @@ class Entry_section(Entry):
self.size, len(data)))
return data
- def ReadChildData(self, child, decomp=True):
+ def ReadChildData(self, child, decomp=True, alt_format=None):
tout.Debug(f"ReadChildData for child '{child.GetPath()}'")
- parent_data = self.ReadData(True)
+ parent_data = self.ReadData(True, alt_format)
offset = child.offset - self._skip_at_start
tout.Debug("Extract for child '%s': offset %#x, skip_at_start %#x, result %#x" %
(child.GetPath(), child.offset, self._skip_at_start, offset))
@@ -775,6 +780,10 @@ class Entry_section(Entry):
tout.Info("%s: Decompressing data size %#x with algo '%s' to data size %#x" %
(child.GetPath(), len(indata), child.compress,
len(data)))
+ if alt_format:
+ new_data = child.GetAltFormat(data, alt_format)
+ if new_data is not None:
+ data = new_data
return data
def WriteChildData(self, child):
@@ -846,3 +855,7 @@ class Entry_section(Entry):
if not self._ignore_missing:
missing = ', '.join(missing)
entry.Raise(f'Missing required properties/entry args: {missing}')
+
+ def CheckAltFormats(self, alt_formats):
+ for entry in self._entries.values():
+ entry.CheckAltFormats(alt_formats)
diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py
index d8ddbc81c1f..0ce829d1225 100644
--- a/tools/binman/ftest.py
+++ b/tools/binman/ftest.py
@@ -4898,6 +4898,40 @@ class TestFunctional(unittest.TestCase):
# There should be a U-Boot after the final FIP
self.assertEqual(U_BOOT_DATA, data[-4:])
+ def testAltFormat(self):
+ """Test that alternative formats can be used to extract"""
+ self._DoReadFileRealDtb('213_fdtmap_alt_format.dts')
+
+ try:
+ tmpdir, updated_fname = self._SetupImageInTmpdir()
+ with test_util.capture_sys_output() as (stdout, _):
+ self._DoBinman('extract', '-i', updated_fname, '-F', 'list')
+ self.assertEqual(
+ '''Flag (-F) Entry type Description
+fdt fdtmap Extract the devicetree blob from the fdtmap
+''',
+ stdout.getvalue())
+
+ dtb = os.path.join(tmpdir, 'fdt.dtb')
+ self._DoBinman('extract', '-i', updated_fname, '-F', 'fdt', '-f',
+ dtb, 'fdtmap')
+
+ # Check that we can read it and it can be scanning, meaning it does
+ # not have a 16-byte fdtmap header
+ data = tools.ReadFile(dtb)
+ dtb = fdt.Fdt.FromData(data)
+ dtb.Scan()
+
+ # Now check u-boot which has no alt_format
+ fname = os.path.join(tmpdir, 'fdt.dtb')
+ self._DoBinman('extract', '-i', updated_fname, '-F', 'dummy',
+ '-f', fname, 'u-boot')
+ data = tools.ReadFile(fname)
+ self.assertEqual(U_BOOT_DATA, data)
+
+ finally:
+ shutil.rmtree(tmpdir)
+
if __name__ == "__main__":
unittest.main()
diff --git a/tools/binman/image.py b/tools/binman/image.py
index 891e8b488e9..f0a7d65299e 100644
--- a/tools/binman/image.py
+++ b/tools/binman/image.py
@@ -223,7 +223,7 @@ class Image(section.Entry_section):
entries = entry.GetEntries()
return entry
- def ReadData(self, decomp=True):
+ def ReadData(self, decomp=True, alt_format=None):
tout.Debug("Image '%s' ReadData(), size=%#x" %
(self.GetPath(), len(self._data)))
return self._data
diff --git a/tools/binman/test/213_fdtmap_alt_format.dts b/tools/binman/test/213_fdtmap_alt_format.dts
new file mode 100644
index 00000000000..d9aef04bcf6
--- /dev/null
+++ b/tools/binman/test/213_fdtmap_alt_format.dts
@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+ #address-cells = <1>;
+ #size-cells = <1>;
+
+ binman {
+ u-boot {
+ };
+ fdtmap {
+ };
+ };
+};
diff --git a/tools/binman/test/214_no_alt_format.dts b/tools/binman/test/214_no_alt_format.dts
new file mode 100644
index 00000000000..f00bcdd5764
--- /dev/null
+++ b/tools/binman/test/214_no_alt_format.dts
@@ -0,0 +1,13 @@
+// SPDX-License-Identifier: GPL-2.0+
+
+/dts-v1/;
+
+/ {
+ #address-cells = <1>;
+ #size-cells = <1>;
+
+ binman {
+ u-boot {
+ };
+ };
+};
--
2.34.0.rc2.393.gf8c9666880-goog
More information about the U-Boot
mailing list