[PATCH v3 01/76] moveconfig: Add an option to compare Kconfig against source

Simon Glass sjg at chromium.org
Wed Feb 1 21:19:12 CET 2023


Sometimes the Makefile rules or source code refers to Kconfig options
which don't exist. Update the moveconfig tool to check this and produce
a series of reports about inconsistencies.

This can then be used to generate patches to correct the problems.

Signed-off-by: Simon Glass <sjg at chromium.org>
---

Changes in v3:
- Update to support lower case CONFIG options
- Drop efi_loader change

 tools/moveconfig.py | 334 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 334 insertions(+)

diff --git a/tools/moveconfig.py b/tools/moveconfig.py
index 8f084a6070a..c4d72ede368 100755
--- a/tools/moveconfig.py
+++ b/tools/moveconfig.py
@@ -93,6 +93,18 @@ SIZES = {
 
 RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig')
 
+# CONFIG symbols present in the build system (from Linux) but not actually used
+# in U-Boot; KCONFIG symbols
+IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL',
+               'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL',
+               'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT',
+               'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3',
+               'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO',
+               'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING',
+               'VAL', '_UNDEFINED', 'SPL_BUILD', ]
+
+SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_']
+
 ### helper functions ###
 def check_top_directory():
     """Exit if we are not at the top of source directory."""
@@ -536,6 +548,7 @@ class KconfigScanner:
         os.environ['srctree'] = os.getcwd()
         os.environ['UBOOTVERSION'] = 'dummy'
         os.environ['KCONFIG_OBJDIR'] = ''
+        os.environ['CC'] = 'gcc'
         self.conf = kconfiglib.Kconfig()
 
 
@@ -1593,6 +1606,319 @@ def prefix_config(cfg):
     return op + cfg
 
 
+RE_MK_CONFIGS = re.compile('CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)')
+RE_IFDEF = re.compile('(ifdef|ifndef)')
+RE_C_CONFIGS = re.compile('CONFIG_([A-Za-z0-9_]*)')
+RE_CONFIG_IS = re.compile('CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)')
+
+class ConfigUse:
+    def __init__(self, cfg, is_spl, fname, rest):
+        self.cfg = cfg
+        self.is_spl = is_spl
+        self.fname = fname
+        self.rest = rest
+
+    def __hash__(self):
+        return hash((self.cfg, self.is_spl))
+
+def scan_makefiles(fnames):
+    """Scan Makefiles looking for Kconfig options
+
+    Looks for uses of CONFIG options in Makefiles
+
+    Args:
+        fnames (list of tuple):
+            str: Makefile filename where the option was found
+            str: Line of the Makefile
+
+    Returns:
+        tuple:
+            dict: all_uses
+                key (ConfigUse): object
+                value (list of str): matching lines
+            dict: Uses by filename
+                key (str): filename
+                value (set of ConfigUse): uses in that filename
+
+    >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups()
+    (None, 'FRED')
+    >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups()
+    ('$(SPL_)', 'MARY')
+    >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups()
+    ('$(SPL_TPL_)', 'MARY')
+    """
+    all_uses = collections.defaultdict(list)
+    fname_uses = {}
+    for fname, rest in fnames:
+        m_iter = RE_MK_CONFIGS.finditer(rest)
+        found = False
+        for m in m_iter:
+            found = True
+            real_opt = m.group(2)
+            if real_opt == '':
+                continue
+            is_spl = False
+            if m.group(1):
+                is_spl = True
+            use = ConfigUse(real_opt, is_spl, fname, rest)
+            if fname not in fname_uses:
+                fname_uses[fname] = set()
+            fname_uses[fname].add(use)
+            all_uses[use].append(rest)
+    return all_uses, fname_uses
+
+
+def scan_src_files(fnames):
+    """Scan source files (other than Makefiles) looking for Kconfig options
+
+    Looks for uses of CONFIG options
+
+    Args:
+        fnames (list of tuple):
+            str: Makefile filename where the option was found
+            str: Line of the Makefile
+
+    Returns:
+        tuple:
+            dict: all_uses
+                key (ConfigUse): object
+                value (list of str): matching lines
+            dict: Uses by filename
+                key (str): filename
+                value (set of ConfigUse): uses in that filename
+
+    >>> RE_C_CONFIGS.search('CONFIG_FRED').groups()
+    ('FRED',)
+    >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups()
+    ('MARY',)
+    >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups()
+    ('OF_PLATDATA',)
+    """
+    def add_uses(m_iter, is_spl):
+        for m in m_iter:
+            found = True
+            real_opt = m.group(1)
+            if real_opt == '':
+                continue
+            use = ConfigUse(real_opt, is_spl, fname, rest)
+            if fname not in fname_uses:
+                fname_uses[fname] = set()
+            fname_uses[fname].add(use)
+            all_uses[use].append(rest)
+
+    all_uses = collections.defaultdict(list)
+    fname_uses = {}
+    for fname, rest in fnames:
+        m_iter = RE_C_CONFIGS.finditer(rest)
+        add_uses(m_iter, False)
+
+        m_iter2 = RE_CONFIG_IS.finditer(rest)
+        add_uses(m_iter2, True)
+
+    return all_uses, fname_uses
+
+
+MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3)
+
+def do_scan_source(path, do_update):
+    """Scan the source tree for Kconfig inconsistencies
+
+    Args:
+        path (str): Path to source tree
+        do_update (bool) : True to write to scripts/kconf_... files
+    """
+    def is_not_proper(name):
+        for prefix in SPL_PREFIXES:
+            if name.startswith(prefix):
+                return name[len(prefix):]
+        return False
+
+    def check_not_found(all_uses, spl_mode):
+        """Check for Kconfig options mentioned in the source but not in Kconfig
+
+        Args:
+            all_uses (dict):
+                key (ConfigUse): object
+                value (list of str): matching lines
+            spl_mode (int): If MODE_SPL, look at source code which implies
+                an SPL_ option, but for which there is none;
+                for MOD_PROPER, look at source code which implies a Proper
+                option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or
+                $(SPL_TPL_) but for which there none;
+                if MODE_NORMAL, ignore SPL
+
+        Returns:
+            dict:
+                key (str): CONFIG name (without 'CONFIG_' prefix
+                value (list of ConfigUse): List of uses of this CONFIG
+        """
+        # Make sure we know about all the options
+        not_found = collections.defaultdict(list)
+        for use, rest in all_uses.items():
+            name = use.cfg
+            if name in IGNORE_SYMS:
+                continue
+            check = True
+
+            if spl_mode == MODE_SPL:
+                check = use.is_spl
+
+                # If it is an SPL symbol, try prepending all SPL_ prefixes to
+                # find at least one SPL symbol
+                if use.is_spl:
+                    add_to_dict = False
+                    for prefix in SPL_PREFIXES:
+                        try_name = prefix + name
+                        sym = kconf.syms.get(try_name)
+                        if sym:
+                            break
+                    if not sym:
+                        not_found[f'SPL_{name}'].append(use)
+                    continue
+            elif spl_mode == MODE_PROPER:
+                # Try to find the Proper version of this symbol, i.e. without
+                # the SPL_ prefix
+                proper_name = is_not_proper(name)
+                if proper_name:
+                    name = proper_name
+                elif not use.is_spl:
+                    check = False
+            else: # MODE_NORMAL
+                debug = False
+                sym = kconf.syms.get(name)
+                if not sym:
+                    proper_name = is_not_proper(name)
+                    if proper_name:
+                        name = proper_name
+                    sym = kconf.syms.get(name)
+                if not sym:
+                    for prefix in SPL_PREFIXES:
+                        try_name = prefix + name
+                        sym = kconf.syms.get(try_name)
+                        if sym:
+                            break
+                if not sym:
+                    not_found[name].append(use)
+                continue
+
+            sym = kconf.syms.get(name)
+            if not sym and check:
+                not_found[name].append(use)
+        return not_found
+
+    def show_uses(uses):
+        """Show a list of uses along with their filename and code snippet
+
+        Args:
+            uses (dict):
+                key (str): CONFIG name (without 'CONFIG_' prefix
+                value (list of ConfigUse): List of uses of this CONFIG
+        """
+        for name in sorted(uses):
+            print(f'{name}: ', end='')
+            for i, use in enumerate(uses[name]):
+                print(f'{"   " if i else ""}{use.fname}: {use.rest.strip()}')
+
+
+    print('Scanning Kconfig')
+    kconf = KconfigScanner().conf
+    print(f'Scanning source in {path}')
+    args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG']
+    with subprocess.Popen(args, stdout=subprocess.PIPE) as proc:
+        out, err = proc.communicate()
+    lines = out.splitlines()
+    re_fname = re.compile('^([^:]*):(.*)')
+    src_list = []
+    mk_list = []
+    for line in lines:
+        linestr = line.decode('utf-8')
+        m_fname = re_fname.search(linestr)
+        if not m_fname:
+            continue
+        fname, rest = m_fname.groups()
+        dirname, leaf = os.path.split(fname)
+        root, ext = os.path.splitext(leaf)
+        if ext == '.autoconf':
+            pass
+        elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg',
+                     '.env', '.tmpl']:
+            src_list.append([fname, rest])
+        elif 'Makefile' in root or ext == '.mk':
+            mk_list.append([fname, rest])
+        elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']:
+            pass
+        elif 'Kconfig' in root or 'Kbuild' in root:
+            pass
+        elif 'README' in root:
+            pass
+        elif dirname in ['configs']:
+            pass
+        elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'):
+            pass
+        else:
+            print(f'Not sure how to handle file {fname}')
+
+    # Scan the Makefiles
+    all_uses, fname_uses = scan_makefiles(mk_list)
+
+    spl_not_found = set()
+    proper_not_found = set()
+
+    # Make sure we know about all the options
+    print('\nCONFIG options present in Makefiles but not Kconfig:')
+    not_found = check_not_found(all_uses, MODE_NORMAL)
+    show_uses(not_found)
+
+    print('\nCONFIG options present in Makefiles but not Kconfig (SPL):')
+    not_found = check_not_found(all_uses, MODE_SPL)
+    show_uses(not_found)
+    spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
+
+    print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:')
+    not_found = check_not_found(all_uses, MODE_PROPER)
+    show_uses(not_found)
+    proper_not_found |= set([key for key in not_found.keys()])
+
+    # Scan the source code
+    all_uses, fname_uses = scan_src_files(src_list)
+
+    # Make sure we know about all the options
+    print('\nCONFIG options present in source but not Kconfig:')
+    not_found = check_not_found(all_uses, MODE_NORMAL)
+    show_uses(not_found)
+
+    print('\nCONFIG options present in source but not Kconfig (SPL):')
+    not_found = check_not_found(all_uses, MODE_SPL)
+    show_uses(not_found)
+    spl_not_found |= set([is_not_proper(key) or key for key in not_found.keys()])
+
+    print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:')
+    not_found = check_not_found(all_uses, MODE_PROPER)
+    show_uses(not_found)
+    proper_not_found |= set([key for key in not_found.keys()])
+
+    print('\nCONFIG options used as SPL but without an SPL_ variant:')
+    for item in sorted(spl_not_found):
+        print(f'   {item}')
+
+    print('\nCONFIG options used as Proper but without a non-SPL_ variant:')
+    for item in sorted(proper_not_found):
+        print(f'   {item}')
+
+    # Write out the updated information
+    if do_update:
+        with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w') as out:
+            print('# These options should not be enabled in SPL builds\n',
+                  file=out)
+            for item in sorted(spl_not_found):
+                print(item, file=out)
+        with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w') as out:
+            print('# These options should not be enabled in Proper builds\n',
+                  file=out)
+            for item in sorted(proper_not_found):
+                print(item, file=out)
+
+
 def main():
     try:
         cpu_count = multiprocessing.cpu_count()
@@ -1642,10 +1968,14 @@ doc/develop/moveconfig.rst for documentation.'''
                       help='force sync by savedefconfig')
     parser.add_argument('-S', '--spl', action='store_true', default=False,
                       help='parse config options defined for SPL build')
+    parser.add_argument('--scan-source', action='store_true', default=False,
+                      help='scan source for uses of CONFIG options')
     parser.add_argument('-t', '--test', action='store_true', default=False,
                       help='run unit tests')
     parser.add_argument('-y', '--yes', action='store_true', default=False,
                       help="respond 'yes' to any prompts")
+    parser.add_argument('-u', '--update', action='store_true', default=False,
+                      help="update scripts/ files (use with --scan-source)")
     parser.add_argument('-v', '--verbose', action='store_true', default=False,
                       help='show any build errors as boards are built')
     parser.add_argument('configs', nargs='*')
@@ -1660,6 +1990,10 @@ doc/develop/moveconfig.rst for documentation.'''
             return 1
         unittest.main()
 
+    if args.scan_source:
+        do_scan_source(os.getcwd(), args.update)
+        return
+
     if not any((len(configs), args.force_sync, args.build_db, args.imply,
                 args.find)):
         parser.print_usage()
-- 
2.39.1.456.gfc5497dd1b-goog



More information about the U-Boot mailing list