[PATCH v2 3/4] patman: Run get_maintainer.pl in parallel

[Doug Anderson]

Hi,

On Sun, Feb 19, 2023 at 3:50 PM Simon Glass <sjg at chromium.org> wrote:
>
> This script can take ages on some series. Try to limit the time by
> using threads. If a few stubborn patches remain, show progress so the
> user has some idea what is going on.
>
> Signed-off-by: Simon Glass <sjg at chromium.org>
> ---
>
> (no changes since v1)
>
>  tools/patmanu/func_test.py |  2 ++
>  tools/patmanu/series.py    | 33 ++++++++++++++++++++++++++++++---
>  2 files changed, 32 insertions(+), 3 deletions(-)
>
> diff --git a/tools/patmanu/func_test.py b/tools/patmanu/func_test.py
> index 238fd5b6100..48109ae5725 100644
> --- a/tools/patmanu/func_test.py
> +++ b/tools/patmanu/func_test.py
> @@ -240,6 +240,8 @@ class TestFunctional(unittest.TestCase):
>          self.assertEqual('Change log missing for v3', next(lines))
>          self.assertEqual('Change log for unknown version v4', next(lines))
>          self.assertEqual("Alias 'pci' not found", next(lines))
> +        while next(lines) != 'Cc processing complete':
> +            pass
>          self.assertIn('Dry run', next(lines))
>          self.assertEqual('', next(lines))
>          self.assertIn('Send a total of %d patches' % count, next(lines))
> diff --git a/tools/patmanu/series.py b/tools/patmanu/series.py
> index 8ead87ef53e..e7a5f91da87 100644
> --- a/tools/patmanu/series.py
> +++ b/tools/patmanu/series.py
> @@ -5,8 +5,11 @@
>  from __future__ import print_function
>
>  import collections
> +import concurrent.futures
>  import itertools
>  import os
> +import sys
> +import time
>
>  from patmanu import get_maintainer
>  from patmanu import gitutil
> @@ -302,10 +305,34 @@ class Series(dict):
>          fd = open(fname, 'w', encoding='utf-8')
>          all_ccs = []
>          all_skips = set()
> +        with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor:
> +            for i, commit in enumerate(self.commits):
> +                commit.seq = i
> +                commit.future = executor.submit(
> +                    self.GetCcForCommit, commit, process_tags, warn_on_error,
> +                    add_maintainers, limit, get_maintainer_script, all_skips)
> +
> +            # Show progress any commits that are taking forever
> +            lastlen = 0
> +            while True:
> +                left = [commit for commit in self.commits
> +                        if not commit.future.done()]
> +                if not left:
> +                    break
> +                names = ', '.join(f'{c.seq + 1}:{c.subject}'
> +                                  for c in left[:2])
> +                out = f'\r{len(left)} remaining: {names}'[:79]
> +                spaces = ' ' * (lastlen - len(out))
> +                if lastlen:  # Don't print anything the first time
> +                    print(out, spaces, end='')
> +                    sys.stdout.flush()
> +                lastlen = len(out)
> +                time.sleep(.25)
> +            print(f'\rdone{" " * lastlen}\r', end='')
> +            print('Cc processing complete')
> +
>          for commit in self.commits:
> -            cc = self.GetCcForCommit(commit, process_tags, warn_on_error,
> -                                     add_maintainers, limit,
> -                                     get_maintainer_script, all_skips)
> +            cc = commit.future.result()

I've never used "concurrent.futures" before, but looks reasonable to me.

Reviewed-by: Douglas Anderson <dianders at chromium.org>