[U-Boot] [PATCH v3 08/18] buildman: Add a functional test
Simon Glass
sjg at chromium.org
Sat Sep 6 03:00:13 CEST 2014
Buildman currently lacks testing in many areas, including its use of git,
make and many command-line flags.
Add a functional test which covers some of these areas. So far it does
a fake 'build' of all boards for the current source tree.
This version reads the real ~/.buildman and boards.cfg files. Future work
will improve this.
Signed-off-by: Simon Glass <sjg at chromium.org>
---
Changes in v3: None
Changes in v2:
- Add a comment to _HandleCommandGit
- Make sure the test temporary directory is removed
tools/buildman/buildman.py | 17 ++---
tools/buildman/control.py | 21 +++--
tools/buildman/func_test.py | 182 ++++++++++++++++++++++++++++++++++++++++++++
tools/buildman/toolchain.py | 4 +-
4 files changed, 206 insertions(+), 18 deletions(-)
create mode 100644 tools/buildman/func_test.py
diff --git a/tools/buildman/buildman.py b/tools/buildman/buildman.py
index 70c2142..6771c86 100755
--- a/tools/buildman/buildman.py
+++ b/tools/buildman/buildman.py
@@ -30,27 +30,20 @@ import terminal
import toolchain
def RunTests():
+ import func_test
import test
import doctest
result = unittest.TestResult()
- for module in ['toolchain']:
+ for module in ['toolchain', 'gitutil']:
suite = doctest.DocTestSuite(module)
suite.run(result)
- # TODO: Surely we can just 'print' result?
- print result
- for test, err in result.errors:
- print err
- for test, err in result.failures:
- print err
-
sys.argv = [sys.argv[0]]
- suite = unittest.TestLoader().loadTestsFromTestCase(test.TestBuild)
- result = unittest.TestResult()
- suite.run(result)
+ for module in (test.TestBuild, func_test.TestFunctional):
+ suite = unittest.TestLoader().loadTestsFromTestCase(module)
+ suite.run(result)
- # TODO: Surely we can just 'print' result?
print result
for test, err in result.errors:
print err
diff --git a/tools/buildman/control.py b/tools/buildman/control.py
index 408d9b1..213e235 100644
--- a/tools/buildman/control.py
+++ b/tools/buildman/control.py
@@ -13,6 +13,7 @@ from builder import Builder
import gitutil
import patchstream
import terminal
+from terminal import Print
import toolchain
import command
import subprocess
@@ -77,12 +78,18 @@ def ShowActions(series, why_selected, boards_selected, builder, options):
print ('Total boards to build for each commit: %d\n' %
why_selected['all'])
-def DoBuildman(options, args):
+def DoBuildman(options, args, toolchains=None, make_func=None):
"""The main control code for buildman
Args:
options: Command line options object
args: Command line arguments (list of strings)
+ toolchains: Toolchains to use - this should be a Toolchains()
+ object. If None, then it will be created and scanned
+ make_func: Make function to use for the builder. This is called
+ to execute 'make'. If this is None, the normal function
+ will be used, which calls the 'make' tool with suitable
+ arguments. This setting is useful for tests.
"""
if options.full_help:
pager = os.getenv('PAGER')
@@ -97,8 +104,10 @@ def DoBuildman(options, args):
bsettings.Setup(options.config_file)
options.git_dir = os.path.join(options.git, '.git')
- toolchains = toolchain.Toolchains()
- toolchains.Scan(options.list_tool_chains)
+ if not toolchains:
+ toolchains = toolchain.Toolchains()
+ toolchains.GetSettings()
+ toolchains.Scan(options.list_tool_chains)
if options.list_tool_chains:
toolchains.List()
print
@@ -202,6 +211,8 @@ def DoBuildman(options, args):
options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
show_unknown=options.show_unknown, step=options.step)
builder.force_config_on_failure = not options.quick
+ if make_func:
+ builder.do_make = make_func
# For a dry run, just show our actions as a sanity check
if options.dry_run:
@@ -220,8 +231,8 @@ def DoBuildman(options, args):
else:
commits = None
- print GetActionSummary(options.summary, commits, board_selected,
- options)
+ Print(GetActionSummary(options.summary, commits, board_selected,
+ options))
builder.SetDisplayOptions(options.show_errors, options.show_sizes,
options.show_detail, options.show_bloat,
diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
new file mode 100644
index 0000000..8711f9c
--- /dev/null
+++ b/tools/buildman/func_test.py
@@ -0,0 +1,182 @@
+#
+# Copyright (c) 2014 Google, Inc
+#
+# SPDX-License-Identifier: GPL-2.0+
+#
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+
+import cmdline
+import command
+import control
+import gitutil
+import terminal
+import toolchain
+
+class TestFunctional(unittest.TestCase):
+ """Functional test for buildman.
+
+ This aims to test from just below the invocation of buildman (parsing
+ of arguments) to 'make' and 'git' invocation. It is not a true
+ emd-to-end test, as it mocks git, make and the tool chain. But this
+ makes it easier to detect when the builder is doing the wrong thing,
+ since in many cases this test code will fail. For example, only a
+ very limited subset of 'git' arguments is supported - anything
+ unexpected will fail.
+ """
+ def setUp(self):
+ self._base_dir = tempfile.mkdtemp()
+ self._git_dir = os.path.join(self._base_dir, 'src')
+ self._buildman_pathname = sys.argv[0]
+ self._buildman_dir = os.path.dirname(sys.argv[0])
+ command.test_result = self._HandleCommand
+ self._toolchains = toolchain.Toolchains()
+ self._toolchains.Add('gcc', test=False)
+
+ def tearDown(self):
+ shutil.rmtree(self._base_dir)
+
+ def _RunBuildman(self, *args):
+ return command.RunPipe([[self._buildman_pathname] + list(args)],
+ capture=True, capture_stderr=True)
+
+ def _RunControl(self, *args):
+ sys.argv = [sys.argv[0]] + list(args)
+ options, args = cmdline.ParseArgs()
+ return control.DoBuildman(options, args, toolchains=self._toolchains,
+ make_func=self._HandleMake)
+
+ def testFullHelp(self):
+ command.test_result = None
+ result = self._RunBuildman('-H')
+ help_file = os.path.join(self._buildman_dir, 'README')
+ self.assertEqual(len(result.stdout), os.path.getsize(help_file))
+ self.assertEqual(0, len(result.stderr))
+ self.assertEqual(0, result.return_code)
+
+ def testHelp(self):
+ command.test_result = None
+ result = self._RunBuildman('-h')
+ help_file = os.path.join(self._buildman_dir, 'README')
+ self.assertTrue(len(result.stdout) > 1000)
+ self.assertEqual(0, len(result.stderr))
+ self.assertEqual(0, result.return_code)
+
+ def testGitSetup(self):
+ """Test gitutils.Setup(), from outside the module itself"""
+ command.test_result = command.CommandResult(return_code=1)
+ gitutil.Setup()
+ self.assertEqual(gitutil.use_no_decorate, False)
+
+ command.test_result = command.CommandResult(return_code=0)
+ gitutil.Setup()
+ self.assertEqual(gitutil.use_no_decorate, True)
+
+ def _HandleCommandGitLog(self, args):
+ if '-n0' in args:
+ return command.CommandResult(return_code=0)
+
+ # Not handled, so abort
+ print 'git log', args
+ sys.exit(1)
+
+ def _HandleCommandGit(self, in_args):
+ """Handle execution of a git command
+
+ This uses a hacked-up parser.
+
+ Args:
+ in_args: Arguments after 'git' from the command line
+ """
+ git_args = [] # Top-level arguments to git itself
+ sub_cmd = None # Git sub-command selected
+ args = [] # Arguments to the git sub-command
+ for arg in in_args:
+ if sub_cmd:
+ args.append(arg)
+ elif arg[0] == '-':
+ git_args.append(arg)
+ else:
+ sub_cmd = arg
+ if sub_cmd == 'config':
+ return command.CommandResult(return_code=0)
+ elif sub_cmd == 'log':
+ return self._HandleCommandGitLog(args)
+
+ # Not handled, so abort
+ print 'git', git_args, sub_cmd, args
+ sys.exit(1)
+
+ def _HandleCommandNm(self, args):
+ return command.CommandResult(return_code=0)
+
+ def _HandleCommandObjdump(self, args):
+ return command.CommandResult(return_code=0)
+
+ def _HandleCommandSize(self, args):
+ return command.CommandResult(return_code=0)
+
+ def _HandleCommand(self, **kwargs):
+ """Handle a command execution.
+
+ The command is in kwargs['pipe-list'], as a list of pipes, each a
+ list of commands. The command should be emulated as required for
+ testing purposes.
+
+ Returns:
+ A CommandResult object
+ """
+ pipe_list = kwargs['pipe_list']
+ if len(pipe_list) != 1:
+ print 'invalid pipe', kwargs
+ sys.exit(1)
+ cmd = pipe_list[0][0]
+ args = pipe_list[0][1:]
+ if cmd == 'git':
+ return self._HandleCommandGit(args)
+ elif cmd == './scripts/show-gnu-make':
+ return command.CommandResult(return_code=0, stdout='make')
+ elif cmd == 'nm':
+ return self._HandleCommandNm(args)
+ elif cmd == 'objdump':
+ return self._HandleCommandObjdump(args)
+ elif cmd == 'size':
+ return self._HandleCommandSize(args)
+
+ # Not handled, so abort
+ print 'unknown command', kwargs
+ sys.exit(1)
+ return command.CommandResult(return_code=0)
+
+ def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
+ """Handle execution of 'make'
+
+ Args:
+ commit: Commit object that is being built
+ brd: Board object that is being built
+ stage: Stage that we are at (mrproper, config, build)
+ cwd: Directory where make should be run
+ args: Arguments to pass to make
+ kwargs: Arguments to pass to command.RunPipe()
+ """
+ if stage == 'mrproper':
+ return command.CommandResult(return_code=0)
+ elif stage == 'config':
+ return command.CommandResult(return_code=0,
+ combined='Test configuration complete')
+ elif stage == 'build':
+ return command.CommandResult(return_code=0)
+
+ # Not handled, so abort
+ print 'make', stage
+ sys.exit(1)
+
+ def testCurrentSource(self):
+ """Very simple test to invoke buildman on the current source"""
+ self._RunControl()
+ lines = terminal.GetPrintTestLines()
+ self.assertTrue(lines[0].text.startswith('Building current source'))
diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py
index 0e91294..27dc318 100644
--- a/tools/buildman/toolchain.py
+++ b/tools/buildman/toolchain.py
@@ -99,6 +99,9 @@ class Toolchains:
def __init__(self):
self.toolchains = {}
self.paths = []
+ self._make_flags = dict(bsettings.GetItems('make-flags'))
+
+ def GetSettings(self):
toolchains = bsettings.GetItems('toolchain')
if not toolchains:
print ("Warning: No tool chains - please add a [toolchain] section"
@@ -110,7 +113,6 @@ class Toolchains:
self.paths += glob.glob(value)
else:
self.paths.append(value)
- self._make_flags = dict(bsettings.GetItems('make-flags'))
def Add(self, fname, test=True, verbose=False):
"""Add a toolchain to our list
--
2.1.0.rc2.206.gedb03e5
More information about the U-Boot
mailing list