[git-webkit] Setup should run git fetch --prune --all

https://bugs.webkit.org/show_bug.cgi?id=253091
rdar://106371364

Reviewed by NOBODY (OOPS!).

During 'git-webkit setup', we should run the equivalent of `git fetch --prune --all`
so that all remotes, although canonical remotes in particular, are up to date.

* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/git.py: Add 'git remote'.
* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/program/setup.py:
(_fetch): Stand-alone 'fetch' function for multi-processing.
(Setup.fetch): Fetch all remotes in a repository.
* Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/setup_unittest.py:

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/mocks/local/git.py

@@ -249,8 +249,13 @@ def __init__(
             ), mocks.Subprocess.Route(
                 self.executable, 'remote', 'add', re.compile(r'.+'),
                 cwd=self.path,
-                completion=mocks.ProcessCompletion(
+                generator=lambda *args, **kwargs: self.add_remote(args[3]),
+            ), mocks.Subprocess.Route(
+                self.executable, 'remote',
+                cwd=self.path,
+                generator=lambda *args, **kwargs: mocks.ProcessCompletion(
                     returncode=0,
+                    stdout='\n'.join(set([key.split('/')[0] for key in self.remotes.keys()])),
                 ),
             ), mocks.Subprocess.Route(
                 self.executable, 'branch', '-a', '--format', '.+', '--merged', '.+',
@@ -1418,3 +1423,10 @@ def is_ancestor(self, ancestor, descendent):
                 )
 
         return mocks.ProcessCompletion(returncode=0 if ancestor in self.rev_list(descendent)else 1)
+
+    def add_remote(self, name):
+        for existing in list(self.remotes.keys()):
+            remote, branch = existing.split('/', 1)
+            if remote == 'origin':
+                self.remotes['{}/{}'.format(name, branch)] = self.remotes[existing][:]
+        return mocks.ProcessCompletion(returncode=0)

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/program/setup.py

@@ -27,13 +27,18 @@
 
 from .command import Command
 from .install_hooks import InstallHooks
-from webkitcorepy import arguments, run, CallByNeed, Editor, OutputCapture, Terminal
-from webkitscmpy import log, local, remote as wspremote
+from webkitcorepy import arguments, run, string_utils, CallByNeed, Editor, OutputCapture, TaskPool, Terminal
+from webkitscmpy import log, local, mocks, remote as wspremote
 
 requests = CallByNeed(lambda: __import__('requests'))
 HTTPBasicAuth = CallByNeed(lambda: __import__('requests.auth', fromlist=['HTTPBasicAuth']).HTTPBasicAuth)
 
 
+def _fetch(path, remote):
+    log.info('    Fetching {}...'.format(remote))
+    return run([local.Git.executable(), 'fetch', '--prune', remote], cwd=path, capture_output=True).returncode
+
+
 class Setup(Command):
     name = 'setup'
     help = 'Configure local settings for the current repository'
@@ -72,6 +77,31 @@ def block_secrets(cls, repository):
         log.info('Enabled secret scanning on {}!'.format(repository.url))
         return True
 
+    @classmethod
+    def fetch(cls, repository):
+        kwargs = dict()
+        if sys.version_info >= (3, 6):
+            kwargs = dict(encoding='utf-8')
+        remote_cmd = run(
+            [repository.executable(), 'remote'],
+            capture_output=True, cwd=repository.root_path,
+            **kwargs
+        )
+        if remote_cmd.returncode:
+            sys.stderr.write('Failed to list remotes in repository\n')
+            sys.stderr.write(remote_cmd.stderr)
+            return remote_cmd.returncode
+        remotes = remote_cmd.stdout.split()
+
+        log.warning('Fetching {}...'.format(string_utils.pluralize(len(remotes), 'remote')))
+        results = []
+        with TaskPool(workers=(1 if mocks.local.Git.top else 12)) as pool:
+            for remote in remotes:
+                pool.do(_fetch, repository.path, remote, callback=lambda value: results.append(value))
+            pool.wait()
+        log.warning('Fetched {}!'.format(string_utils.pluralize(len([n for n in results if not n]), 'remote')))
+        return 1 if any(results) else 0
+
     @classmethod
     def github(cls, args, repository, additional_setup=None, remote=None, team=None, secrets_scanning=None, **kwargs):
         log.info('Saving GitHub credentials in system credential store...')
@@ -546,7 +576,7 @@ def git(cls, args, repository, additional_setup=None, hooks=None, **kwargs):
                 sys.stderr.write(warning)
 
         if not forking or forking == 'No':
-            return result
+            return result | cls.fetch(repository)
 
         secrets_scanning_config_value = repository.config().get('webkitscmpy.remotes.origin.secrets-scanning'.format(name), None)
         if cls.github(
@@ -562,16 +592,7 @@ def git(cls, args, repository, additional_setup=None, hooks=None, **kwargs):
             result += 1
         else:
             available_remotes.append('fork')
-
-        for rem in available_remotes:
-            if 'fork' not in rem:
-                continue
-            log.info("Fetching '{}'".format(rem))
-            result += run(
-                [repository.executable(), 'fetch', rem],
-                capture_output=True, cwd=repository.root_path,
-            ).returncode
-        return result
+        return result | cls.fetch(repository)
 
     @classmethod
     def parser(cls, parser, loggers=None):

Tools/Scripts/libraries/webkitscmpy/webkitscmpy/test/setup_unittest.py

@@ -116,6 +116,9 @@ def test_git(self):
 Setting git editor for {repository}...
 Setting contents of 'SVN_LOG_EDITOR' as editor
 Set git editor to 'SVN_LOG_EDITOR' for this repository
+Fetching 1 remote...
+    Fetching origin...
+Fetched 1 remote!
 '''.format(repository=self.path),
         )
 
@@ -190,7 +193,10 @@ def test_github_checkout(self):
 Enabled secret scanning on https://github.example.com/username/WebKit!
 Adding forked remote as 'fork'...
 Added remote 'fork'
-Fetching 'fork'
+Fetching 2 remotes...
+    Fetching origin...
+    Fetching fork...
+Fetched 2 remotes!
 '''.format(repository=self.path),
         )