From b6ef3d81b3028085c37bb1049708045783bd1da9 Mon Sep 17 00:00:00 2001 From: Charles Zablit Date: Wed, 29 Oct 2025 11:46:19 -0700 Subject: [PATCH] [NFC][update-checkout] run the black formatter on update-checkout --- utils/update_checkout/__init__.py | 1 - utils/update_checkout/run_tests.py | 4 +- utils/update_checkout/tests/scheme_mock.py | 171 +++++----- utils/update_checkout/tests/test_clone.py | 65 ++-- utils/update_checkout/tests/test_dump.py | 28 +- .../tests/test_merge_config.py | 62 ++-- .../tests/test_update_worktree.py | 34 +- .../update_checkout/parallel_runner.py | 17 +- .../update_checkout/runner_arguments.py | 3 + .../update_checkout/update_checkout.py | 296 +++++++++++------- 10 files changed, 417 insertions(+), 264 deletions(-) diff --git a/utils/update_checkout/__init__.py b/utils/update_checkout/__init__.py index 99cc1c7c3d1d1..7a4ee69672705 100644 --- a/utils/update_checkout/__init__.py +++ b/utils/update_checkout/__init__.py @@ -1,4 +1,3 @@ - from .update_checkout import main __all__ = ["main"] diff --git a/utils/update_checkout/run_tests.py b/utils/update_checkout/run_tests.py index bd949ad50b023..327bcfd14966c 100755 --- a/utils/update_checkout/run_tests.py +++ b/utils/update_checkout/run_tests.py @@ -24,13 +24,13 @@ UTILS_DIR = os.path.abspath(os.path.join(MODULE_DIR, os.pardir)) -if __name__ == '__main__': +if __name__ == "__main__": # Add the swift/utils directory to the Python path. sys.path.append(UTILS_DIR) # Create temp directory and export it for the test suite. temp_dir = tempfile.mkdtemp() - os.environ['UPDATECHECKOUT_TEST_WORKSPACE_DIR'] = temp_dir + os.environ["UPDATECHECKOUT_TEST_WORKSPACE_DIR"] = temp_dir # Discover all tests for the module. module_tests = unittest.defaultTestLoader.discover(MODULE_DIR) diff --git a/utils/update_checkout/tests/scheme_mock.py b/utils/update_checkout/tests/scheme_mock.py index e703b995e6e29..7c0f7682332b2 100644 --- a/utils/update_checkout/tests/scheme_mock.py +++ b/utils/update_checkout/tests/scheme_mock.py @@ -21,17 +21,17 @@ # For now we only use a config with a single scheme. We should add support for # handling multiple schemes. MOCK_REMOTE = { - 'repo1': [ + "repo1": [ # This is a series of changes to repo1. (File, NewContents) - ('A.txt', 'A'), - ('B.txt', 'B'), - ('A.txt', 'a'), + ("A.txt", "A"), + ("B.txt", "B"), + ("A.txt", "a"), ], - 'repo2': [ + "repo2": [ # This is a series of changes to repo2. (File, NewContents) - ('X.txt', 'X'), - ('Y.txt', 'Y'), - ('X.txt', 'z'), + ("X.txt", "X"), + ("Y.txt", "Y"), + ("X.txt", "z"), ], } @@ -39,40 +39,40 @@ # This is here just b/c we expect it. We should consider consolidating # clone-patterns into a dictionary where we map protocols (i.e. ['ssh, # 'https'] to patterns). Then we can define this issue. - 'ssh-clone-pattern': 'DO_NOT_USE', + "ssh-clone-pattern": "DO_NOT_USE", # We reset this value with our remote path when we process - 'https-clone-pattern': '', - 'repos': { - 'repo1': { - 'remote': {'id': 'repo1'}, + "https-clone-pattern": "", + "repos": { + "repo1": { + "remote": {"id": "repo1"}, }, - 'repo2': { - 'remote': {'id': 'repo2'}, + "repo2": { + "remote": {"id": "repo2"}, }, }, - 'default-branch-scheme': 'main', - 'branch-schemes': { - 'main': { - 'aliases': ['main'], - 'repos': { - 'repo1': 'main', - 'repo2': 'main', - } + "default-branch-scheme": "main", + "branch-schemes": { + "main": { + "aliases": ["main"], + "repos": { + "repo1": "main", + "repo2": "main", + }, } - } + }, } MOCK_ADDITIONAL_SCHEME = { - 'branch-schemes': { - 'extra': { - 'aliases': ['extra'], - 'repos': { + "branch-schemes": { + "extra": { + "aliases": ["extra"], + "repos": { # Spell this differently just to make it distinguishable in # test output, even though we only have one branch. # TODO: Support multiple test branches in the repo instead. - 'repo1': 'refs/heads/main', - 'repo2': 'refs/heads/main', - } + "repo1": "refs/heads/main", + "repo2": "refs/heads/main", + }, } } } @@ -85,18 +85,21 @@ def __init__(self, command, returncode, output): self.output = output def __str__(self): - return f"Command returned a non-zero exit status {self.returncode}:\n"\ - f"Command: {' '.join(self.command)}\n" \ - f"Output: {self.output.decode('utf-8')}" + return ( + f"Command returned a non-zero exit status {self.returncode}:\n" + f"Command: {' '.join(self.command)}\n" + f"Output: {self.output.decode('utf-8')}" + ) def call_quietly(*args, **kwargs): - kwargs['stderr'] = subprocess.STDOUT + kwargs["stderr"] = subprocess.STDOUT try: return subprocess.check_output(*args, **kwargs) except subprocess.CalledProcessError as e: - raise CallQuietlyException(command=e.cmd, returncode=e.returncode, - output=e.stdout) from e + raise CallQuietlyException( + command=e.cmd, returncode=e.returncode, output=e.stdout + ) from e def create_dir(d): @@ -105,75 +108,82 @@ def create_dir(d): def teardown_mock_remote(base_dir): - call_quietly(['rm', '-rf', base_dir]) + call_quietly(["rm", "-rf", base_dir]) def get_config_path(base_dir): - return os.path.join(base_dir, 'test-config.json') + return os.path.join(base_dir, "test-config.json") def get_additional_config_path(base_dir): - return os.path.join(base_dir, 'test-additional-config.json') + return os.path.join(base_dir, "test-additional-config.json") def setup_mock_remote(base_dir, base_config): create_dir(base_dir) # We use local as a workspace for creating commits. - LOCAL_PATH = os.path.join(base_dir, 'local') + LOCAL_PATH = os.path.join(base_dir, "local") # We use remote as a directory that simulates our remote unchecked out # repo. - REMOTE_PATH = os.path.join(base_dir, 'remote') + REMOTE_PATH = os.path.join(base_dir, "remote") create_dir(REMOTE_PATH) create_dir(LOCAL_PATH) - for (k, v) in MOCK_REMOTE.items(): + for k, v in MOCK_REMOTE.items(): local_repo_path = os.path.join(LOCAL_PATH, k) remote_repo_path = os.path.join(REMOTE_PATH, k) create_dir(remote_repo_path) create_dir(local_repo_path) - call_quietly(['git', 'init', '--bare', remote_repo_path]) - call_quietly(['git', 'symbolic-ref', 'HEAD', 'refs/heads/main'], - cwd=remote_repo_path) - call_quietly(['git', 'clone', '-l', remote_repo_path, local_repo_path]) - call_quietly(['git', 'config', '--local', 'user.name', 'swift_test'], - cwd=local_repo_path) - call_quietly(['git', 'config', '--local', 'user.email', 'no-reply@swift.org'], - cwd=local_repo_path) - call_quietly(['git', 'config', '--local', 'commit.gpgsign', 'false'], - cwd=local_repo_path) - call_quietly(['git', 'symbolic-ref', 'HEAD', 'refs/heads/main'], - cwd=local_repo_path) - for (i, (filename, contents)) in enumerate(v): + call_quietly(["git", "init", "--bare", remote_repo_path]) + call_quietly( + ["git", "symbolic-ref", "HEAD", "refs/heads/main"], cwd=remote_repo_path + ) + call_quietly(["git", "clone", "-l", remote_repo_path, local_repo_path]) + call_quietly( + ["git", "config", "--local", "user.name", "swift_test"], cwd=local_repo_path + ) + call_quietly( + ["git", "config", "--local", "user.email", "no-reply@swift.org"], + cwd=local_repo_path, + ) + call_quietly( + ["git", "config", "--local", "commit.gpgsign", "false"], cwd=local_repo_path + ) + call_quietly( + ["git", "symbolic-ref", "HEAD", "refs/heads/main"], cwd=local_repo_path + ) + for i, (filename, contents) in enumerate(v): filename_path = os.path.join(local_repo_path, filename) - with open(filename_path, 'w') as f: + with open(filename_path, "w") as f: f.write(contents) - call_quietly(['git', 'add', filename], cwd=local_repo_path) - call_quietly(['git', 'commit', '-m', 'Commit %d' % i], - cwd=local_repo_path) - call_quietly(['git', 'push', 'origin', 'main'], - cwd=local_repo_path) + call_quietly(["git", "add", filename], cwd=local_repo_path) + call_quietly(["git", "commit", "-m", "Commit %d" % i], cwd=local_repo_path) + call_quietly(["git", "push", "origin", "main"], cwd=local_repo_path) - https_clone_pattern = os.path.join('file://%s' % REMOTE_PATH, '%s') - base_config['https-clone-pattern'] = https_clone_pattern + https_clone_pattern = os.path.join("file://%s" % REMOTE_PATH, "%s") + base_config["https-clone-pattern"] = https_clone_pattern - with open(get_config_path(base_dir), 'w') as f: + with open(get_config_path(base_dir), "w") as f: json.dump(base_config, f) - with open(get_additional_config_path(base_dir), 'w') as f: + with open(get_additional_config_path(base_dir), "w") as f: json.dump(MOCK_ADDITIONAL_SCHEME, f) return (LOCAL_PATH, REMOTE_PATH) -BASEDIR_ENV_VAR = 'UPDATECHECKOUT_TEST_WORKSPACE_DIR' +BASEDIR_ENV_VAR = "UPDATECHECKOUT_TEST_WORKSPACE_DIR" CURRENT_FILE_DIR = os.path.dirname(os.path.abspath(__file__)) -UPDATE_CHECKOUT_EXECUTABLE = 'update-checkout.cmd' if os.name == 'nt' else 'update-checkout' -UPDATE_CHECKOUT_PATH = os.path.abspath(os.path.join(CURRENT_FILE_DIR, - os.path.pardir, - os.path.pardir, - UPDATE_CHECKOUT_EXECUTABLE)) +UPDATE_CHECKOUT_EXECUTABLE = ( + "update-checkout.cmd" if os.name == "nt" else "update-checkout" +) +UPDATE_CHECKOUT_PATH = os.path.abspath( + os.path.join( + CURRENT_FILE_DIR, os.path.pardir, os.path.pardir, UPDATE_CHECKOUT_EXECUTABLE + ) +) class SchemeMockTestCase(unittest.TestCase): @@ -184,16 +194,19 @@ def __init__(self, *args, **kwargs): self.config = MOCK_CONFIG.copy() self.workspace = os.getenv(BASEDIR_ENV_VAR) if self.workspace is None: - raise RuntimeError('Misconfigured test suite! Environment ' - 'variable %s must be set!' % BASEDIR_ENV_VAR) + raise RuntimeError( + "Misconfigured test suite! Environment " + "variable %s must be set!" % BASEDIR_ENV_VAR + ) self.config_path = get_config_path(self.workspace) self.additional_config_path = get_additional_config_path(self.workspace) self.update_checkout_path = UPDATE_CHECKOUT_PATH if not os.access(self.update_checkout_path, os.X_OK): - raise RuntimeError('Error! Could not find executable ' - 'update-checkout at path: %s' - % self.update_checkout_path) - self.source_root = os.path.join(self.workspace, 'source_root') + raise RuntimeError( + "Error! Could not find executable " + "update-checkout at path: %s" % self.update_checkout_path + ) + self.source_root = os.path.join(self.workspace, "source_root") def setUp(self): create_dir(self.source_root) @@ -205,7 +218,7 @@ def tearDown(self): teardown_mock_remote(self.workspace) def call(self, *args, **kwargs): - kwargs['cwd'] = self.source_root + kwargs["cwd"] = self.source_root return call_quietly(*args, **kwargs) def get_all_repos(self): diff --git a/utils/update_checkout/tests/test_clone.py b/utils/update_checkout/tests/test_clone.py index 7799be778615e..26635db9f3c71 100644 --- a/utils/update_checkout/tests/test_clone.py +++ b/utils/update_checkout/tests/test_clone.py @@ -22,37 +22,58 @@ def __init__(self, *args, **kwargs): super(CloneTestCase, self).__init__(*args, **kwargs) def test_simple_clone(self): - self.call([self.update_checkout_path, - '--config', self.config_path, - '--source-root', self.source_root, - '--clone']) + self.call( + [ + self.update_checkout_path, + "--config", + self.config_path, + "--source-root", + self.source_root, + "--clone", + ] + ) for repo in self.get_all_repos(): repo_path = os.path.join(self.source_root, repo) self.assertTrue(os.path.isdir(repo_path)) def test_clone_with_additional_scheme(self): - output = self.call([self.update_checkout_path, - '--config', self.config_path, - '--config', self.additional_config_path, - '--source-root', self.source_root, - '--clone', - '--scheme', 'extra', - '--verbose']) + output = self.call( + [ + self.update_checkout_path, + "--config", + self.config_path, + "--config", + self.additional_config_path, + "--source-root", + self.source_root, + "--clone", + "--scheme", + "extra", + "--verbose", + ] + ) # Test that we're actually checking out the 'extra' scheme based on the output self.assertIn("git checkout refs/heads/main", output.decode("utf-8")) def test_manager_not_called_on_long_socket(self): - fake_tmpdir = '/tmp/very/' + '/long' * 20 + '/tmp' - - with mock.patch('tempfile.gettempdir', return_value=fake_tmpdir), \ - mock.patch('multiprocessing.Manager') as mock_manager: - - self.call([self.update_checkout_path, - '--config', self.config_path, - '--source-root', self.source_root, - '--clone']) + fake_tmpdir = "/tmp/very/" + "/long" * 20 + "/tmp" + + with mock.patch("tempfile.gettempdir", return_value=fake_tmpdir), mock.patch( + "multiprocessing.Manager" + ) as mock_manager: + + self.call( + [ + self.update_checkout_path, + "--config", + self.config_path, + "--source-root", + self.source_root, + "--clone", + ] + ) # Ensure that we do not try to create a Manager when the tempdir # is too long. mock_manager.assert_not_called() @@ -85,9 +106,7 @@ def __init__(self, *args, **kwargs): def test_clone(self): self.call(self.base_args + ["--scheme", self.scheme_name, "--clone"]) - missing_repo_path = os.path.join( - self.source_root, self.get_all_repos().pop() - ) + missing_repo_path = os.path.join(self.source_root, self.get_all_repos().pop()) self.assertFalse(os.path.isdir(missing_repo_path)) # Test that we do not update a repository that is not listed in the given diff --git a/utils/update_checkout/tests/test_dump.py b/utils/update_checkout/tests/test_dump.py index 87cb76e81d4d8..eb47c286a7661 100644 --- a/utils/update_checkout/tests/test_dump.py +++ b/utils/update_checkout/tests/test_dump.py @@ -22,16 +22,28 @@ def __init__(self, *args, **kwargs): def test_dump_hashes_json(self): # First do the clone. - self.call([self.update_checkout_path, - '--config', self.config_path, - '--source-root', self.source_root, - '--clone']) + self.call( + [ + self.update_checkout_path, + "--config", + self.config_path, + "--source-root", + self.source_root, + "--clone", + ] + ) # Then dump the hashes. - output = self.call([self.update_checkout_path, - '--config', self.config_path, - '--source-root', self.source_root, - '--dump-hashes']) + output = self.call( + [ + self.update_checkout_path, + "--config", + self.config_path, + "--source-root", + self.source_root, + "--dump-hashes", + ] + ) # The output should be valid JSON result = json.loads(output) diff --git a/utils/update_checkout/tests/test_merge_config.py b/utils/update_checkout/tests/test_merge_config.py index 4963f12b07946..38873d8f93071 100644 --- a/utils/update_checkout/tests/test_merge_config.py +++ b/utils/update_checkout/tests/test_merge_config.py @@ -38,38 +38,44 @@ def test_merge_config(self): }, } - self.assertEqual(merge_config(default_config, { - "note": "this is machine generated or something", - "ssh-clone-pattern": "git@2", - "repos": { - "llvm-project": {"remote": {"id": "blah/llvm-project"}}, - "swift-syntax": {"remote": {"id": "swiftlang/swift-syntax"}}, - }, - "default-branch-scheme": "bonus", - "branch-schemes": { - "bonus": { - "aliases": ["bonus", "also-bonus"], + self.assertEqual( + merge_config( + default_config, + { + "note": "this is machine generated or something", + "ssh-clone-pattern": "git@2", + "repos": { + "llvm-project": {"remote": {"id": "blah/llvm-project"}}, + "swift-syntax": {"remote": {"id": "swiftlang/swift-syntax"}}, + }, + "default-branch-scheme": "bonus", + "branch-schemes": { + "bonus": { + "aliases": ["bonus", "also-bonus"], + }, + }, }, - }, - }), { - "ssh-clone-pattern": "git@2", - "https-clone-pattern": "https://1", - "repos": { - "swift": {"remote": {"id": "swiftlang/swift"}}, - "llvm-project": {"remote": {"id": "blah/llvm-project"}}, - "swift-syntax": {"remote": {"id": "swiftlang/swift-syntax"}}, - }, - "default-branch-scheme": "bonus", - "branch-schemes": { - "main": { - "aliases": ["swift/main", "main", "stable/20240723"], + ), + { + "ssh-clone-pattern": "git@2", + "https-clone-pattern": "https://1", + "repos": { + "swift": {"remote": {"id": "swiftlang/swift"}}, + "llvm-project": {"remote": {"id": "blah/llvm-project"}}, + "swift-syntax": {"remote": {"id": "swiftlang/swift-syntax"}}, }, - "bonus": { - "aliases": ["bonus", "also-bonus"], + "default-branch-scheme": "bonus", + "branch-schemes": { + "main": { + "aliases": ["swift/main", "main", "stable/20240723"], + }, + "bonus": { + "aliases": ["bonus", "also-bonus"], + }, }, + "note": "this is machine generated or something", }, - "note": "this is machine generated or something", - }) + ) with self.assertRaises(ValueError): merge_config(default_config, default_config) diff --git a/utils/update_checkout/tests/test_update_worktree.py b/utils/update_checkout/tests/test_update_worktree.py index a50df1ab063e7..c3cbc1ca9fe03 100644 --- a/utils/update_checkout/tests/test_update_worktree.py +++ b/utils/update_checkout/tests/test_update_worktree.py @@ -29,9 +29,9 @@ def setup_worktree(workspace_path, local_path, worktree_name): for project in os.listdir(local_path): local_project_path = os.path.join(local_path, project) worktree_project_path = os.path.join(worktree_path, project) - call_quietly(['git', - '-C', local_project_path, - 'worktree', 'add', worktree_project_path]) + call_quietly( + ["git", "-C", local_project_path, "worktree", "add", worktree_project_path] + ) def teardown_worktree(workspace_path, local_path, worktree_name): @@ -39,9 +39,16 @@ def teardown_worktree(workspace_path, local_path, worktree_name): for project in os.listdir(local_path): local_project_path = os.path.join(local_path, project) worktree_project_path = os.path.join(worktree_path, project) - call_quietly(['git', - '-C', local_project_path, - 'worktree', 'remove', worktree_project_path]) + call_quietly( + [ + "git", + "-C", + local_project_path, + "worktree", + "remove", + worktree_project_path, + ] + ) class WorktreeTestCase(scheme_mock.SchemeMockTestCase): @@ -50,10 +57,17 @@ def __init__(self, *args, **kwargs): super(WorktreeTestCase, self).__init__(*args, **kwargs) def test_worktree(self): - self.call([self.update_checkout_path, - '--config', self.config_path, - '--source-root', self.worktree_path, - '--scheme', 'main']) + self.call( + [ + self.update_checkout_path, + "--config", + self.config_path, + "--source-root", + self.worktree_path, + "--scheme", + "main", + ] + ) def setUp(self): super(WorktreeTestCase, self).setUp() diff --git a/utils/update_checkout/update_checkout/parallel_runner.py b/utils/update_checkout/update_checkout/parallel_runner.py index edee4145463a3..f00840abc2e46 100644 --- a/utils/update_checkout/update_checkout/parallel_runner.py +++ b/utils/update_checkout/update_checkout/parallel_runner.py @@ -8,7 +8,11 @@ from .git_command import GitException -from .runner_arguments import RunnerArguments, AdditionalSwiftSourcesArguments, UpdateArguments +from .runner_arguments import ( + RunnerArguments, + AdditionalSwiftSourcesArguments, + UpdateArguments, +) class TaskTracker: @@ -71,7 +75,7 @@ def __call__(self, *args: Union[RunnerArguments, AdditionalSwiftSourcesArguments return result -class ParallelRunner(): +class ParallelRunner: def __init__( self, fn: Callable[..., None], @@ -111,7 +115,9 @@ def run(self) -> List[Union[None, Exception]]: monitor_thread = Thread(target=self._monitor, daemon=True) monitor_thread.start() with ThreadPoolExecutor(max_workers=self._n_threads) as pool: - results = list(pool.map(self._monitored_fn, self._pool_args, timeout=1800)) + results = list( + pool.map(self._monitored_fn, self._pool_args, timeout=1800) + ) self._stop_event.set() monitor_thread.join() return results @@ -121,7 +127,10 @@ def _monitor(self): while not self._stop_event.is_set(): current_line, updated_repos = self._task_tracker.status() if current_line != last_output: - truncated = f"{self._output_prefix} [{updated_repos}/{self._nb_repos}] ({current_line})" + truncated = ( + f"{self._output_prefix} [{updated_repos}/{self._nb_repos}] " + f"({current_line})" + ) if len(truncated) > self._terminal_width: ellipsis_marker = " ..." truncated = ( diff --git a/utils/update_checkout/update_checkout/runner_arguments.py b/utils/update_checkout/update_checkout/runner_arguments.py index 2c9cc1a52aaf7..f55c75c8c6f47 100644 --- a/utils/update_checkout/update_checkout/runner_arguments.py +++ b/utils/update_checkout/update_checkout/runner_arguments.py @@ -3,6 +3,7 @@ from .cli_arguments import CliArguments + @dataclass class RunnerArguments: repo_name: str @@ -10,6 +11,7 @@ class RunnerArguments: output_prefix: str verbose: bool + @dataclass class UpdateArguments(RunnerArguments): source_root: str @@ -22,6 +24,7 @@ class UpdateArguments(RunnerArguments): stash: bool cross_repos_pr: Dict[str, str] + @dataclass class AdditionalSwiftSourcesArguments(RunnerArguments): args: CliArguments diff --git a/utils/update_checkout/update_checkout/update_checkout.py b/utils/update_checkout/update_checkout/update_checkout.py index 8204da145e6db..eb57f74cd9ce4 100755 --- a/utils/update_checkout/update_checkout/update_checkout.py +++ b/utils/update_checkout/update_checkout/update_checkout.py @@ -60,13 +60,20 @@ def confirm_tag_in_repo(repo_path: str, tag: str, repo_name: str) -> Optional[st repo_path, ["ls-remote", "--tags", "origin", tag], fatal=True ) if not tag_exists: - print("Tag '" + tag + "' does not exist for '" + - repo_name + "', just updating regularly") + print( + "Tag '" + + tag + + "' does not exist for '" + + repo_name + + "', just updating regularly" + ) return None return tag -def find_rev_by_timestamp(repo_path: str, timestamp: str, repo_name: str, refspec: str) -> str: +def find_rev_by_timestamp( + repo_path: str, timestamp: str, repo_name: str, refspec: str +) -> str: refspec_exists = True try: Git.run(repo_path, ["rev-parse", "--verify", refspec]) @@ -79,8 +86,7 @@ def find_rev_by_timestamp(repo_path: str, timestamp: str, repo_name: str, refspe if rev: return rev else: - raise RuntimeError('No rev in %s before timestamp %s' % - (repo_name, timestamp)) + raise RuntimeError("No rev in %s before timestamp %s" % (repo_name, timestamp)) def get_branch_for_repo( @@ -112,14 +118,29 @@ def get_branch_for_repo( if scheme_map: scheme_branch = scheme_map[repo_name] repo_branch = scheme_branch - remote_repo_id = config['repos'][repo_name]['remote']['id'] + remote_repo_id = config["repos"][repo_name]["remote"]["id"] if remote_repo_id in cross_repos_pr: cross_repo = True pr_id = cross_repos_pr[remote_repo_id] repo_branch = "ci_pr_{0}".format(pr_id) Git.run(repo_path, ["checkout", scheme_branch], echo=True) - Git.run(repo_path, ["branch", "-D", repo_branch], echo=True, allow_non_zero_exit=True, fatal=True) - Git.run(repo_path, ["fetch", "origin", "pull/{0}/merge:{1}".format(pr_id, repo_branch), "--tags"], echo=True) + Git.run( + repo_path, + ["branch", "-D", repo_branch], + echo=True, + allow_non_zero_exit=True, + fatal=True, + ) + Git.run( + repo_path, + [ + "fetch", + "origin", + "pull/{0}/merge:{1}".format(pr_id, repo_branch), + "--tags", + ], + echo=True, + ) return repo_branch, cross_repo @@ -139,9 +160,7 @@ def update_single_repository(pool_args: UpdateArguments): cross_repo = False checkout_target = None if pool_args.tag: - checkout_target = confirm_tag_in_repo( - repo_path, pool_args.tag, repo_name - ) + checkout_target = confirm_tag_in_repo(repo_path, pool_args.tag, repo_name) elif pool_args.scheme_name: checkout_target, cross_repo = get_branch_for_repo( repo_path, @@ -177,9 +196,7 @@ def run_for_repo_and_each_submodule_rec(args: List[str]): run_for_repo_and_each_submodule_rec(["stash", "-u"]) elif pool_args.stash: # Delete tracked changes. - run_for_repo_and_each_submodule_rec( - ["reset", "--hard", "HEAD"] - ) + run_for_repo_and_each_submodule_rec(["reset", "--hard", "HEAD"]) # Delete untracked changes and ignored files. run_for_repo_and_each_submodule_rec(["clean", "-fdx"]) @@ -187,12 +204,12 @@ def run_for_repo_and_each_submodule_rec(args: List[str]): # It is possible to reset --hard and still be mid-rebase. try: - Git.run(repo_path, ['rebase', '--abort'], echo=verbose, prefix=prefix) + Git.run(repo_path, ["rebase", "--abort"], echo=verbose, prefix=prefix) except Exception: pass if checkout_target: - Git.run(repo_path, ['status', '--porcelain', '-uno']) + Git.run(repo_path, ["status", "--porcelain", "-uno"]) # Some of the projects switch branches/tags when they # are updated. Local checkout might not have that tag/branch @@ -271,9 +288,10 @@ def run_for_repo_and_each_submodule_rec(args: List[str]): if not cross_repo and not detached_head: Git.run(repo_path, ["rebase", "FETCH_HEAD"], echo=verbose, prefix=prefix) elif detached_head and verbose: - print(prefix + - "Detached HEAD; probably checked out a tag. No need " - "to rebase.") + print( + prefix + "Detached HEAD; probably checked out a tag. No need " + "to rebase." + ) Git.run( repo_path, @@ -307,7 +325,9 @@ def get_timestamp_to_match(match_timestamp: bool, source_root: str): return output -def get_scheme_map(config: Dict[str, Any], scheme_name: str) -> Optional[Dict[str, str]]: +def get_scheme_map( + config: Dict[str, Any], scheme_name: str +) -> Optional[Dict[str, str]]: """Find a mapping from repository IDs to branches in the config. Args: @@ -323,12 +343,13 @@ def get_scheme_map(config: Dict[str, Any], scheme_name: str) -> Optional[Dict[st # unique contents. This is checked by validate_config. Thus the first # branch scheme data that has scheme_name as one of its aliases is # the only possible correct answer. - for v in config['branch-schemes'].values(): - if scheme_name in v['aliases']: - return v['repos'] + for v in config["branch-schemes"].values(): + if scheme_name in v["aliases"]: + return v["repos"] return None + def _is_any_repository_locked(pool_args: List[UpdateArguments]) -> Set[str]: """Returns the set of locked repositories. @@ -354,7 +375,10 @@ def _is_any_repository_locked(pool_args: List[UpdateArguments]) -> Set[str]: locked_repositories.add(repo_name) return locked_repositories -def _move_llvm_project_to_first_index(pool_args: Union[List[UpdateArguments], List[AdditionalSwiftSourcesArguments]]): + +def _move_llvm_project_to_first_index( + pool_args: Union[List[UpdateArguments], List[AdditionalSwiftSourcesArguments]], +): llvm_project_idx = None for i in range(len(pool_args)): if pool_args[i].repo_name == "llvm-project": @@ -363,6 +387,7 @@ def _move_llvm_project_to_first_index(pool_args: Union[List[UpdateArguments], Li if llvm_project_idx is not None: pool_args.insert(0, pool_args.pop(llvm_project_idx)) + def update_all_repositories( args: CliArguments, config: Dict[str, Any], @@ -373,16 +398,26 @@ def update_all_repositories( skipped_repositories = [] pool_args: List[UpdateArguments] = [] timestamp = get_timestamp_to_match(args.match_timestamp, args.source_root) - for repo_name in config['repos'].keys(): + for repo_name in config["repos"].keys(): if repo_name in args.skip_repository_list: - skipped_repositories.append(SkippedReason(repo_name, "requested by user",)) + skipped_repositories.append( + SkippedReason( + repo_name, + "requested by user", + ) + ) continue # If the repository is not listed in the branch-scheme, skip it. if scheme_map and repo_name not in scheme_map: # If the repository exists locally, notify we are skipping it. if os.path.isdir(os.path.join(args.source_root, repo_name)): - skipped_repositories.append(SkippedReason(repo_name, f"repository not listed in the {scheme_name} branch-scheme")) + skipped_repositories.append( + SkippedReason( + repo_name, + f"repository not listed in the {scheme_name} branch-scheme", + ) + ) continue my_args = UpdateArguments( @@ -409,7 +444,10 @@ def update_all_repositories( for repo_name in locked_repositories ] _move_llvm_project_to_first_index(pool_args) - return skipped_repositories, ParallelRunner(update_single_repository, pool_args, args.n_processes).run() + return ( + skipped_repositories, + ParallelRunner(update_single_repository, pool_args, args.n_processes).run(), + ) def obtain_additional_swift_sources(pool_args: AdditionalSwiftSourcesArguments): @@ -421,27 +459,44 @@ def obtain_additional_swift_sources(pool_args: AdditionalSwiftSourcesArguments): remote = pool_args.remote env = dict(os.environ) - env.update({'GIT_TERMINAL_PROMPT': '0'}) + env.update({"GIT_TERMINAL_PROMPT": "0"}) if verbose: print("Cloning '" + pool_args.repo_name + "'") if args.skip_history: - Git.run(args.source_root, ['clone', '--recursive', '--depth', '1', - '--branch', repo_branch, remote, repo_name] + - (['--no-tags'] if skip_tags else []), - env=env, - echo=verbose) + Git.run( + args.source_root, + [ + "clone", + "--recursive", + "--depth", + "1", + "--branch", + repo_branch, + remote, + repo_name, + ] + + (["--no-tags"] if skip_tags else []), + env=env, + echo=verbose, + ) elif args.use_submodules: - Git.run(args.source_root, ['submodule', 'add', remote, repo_name] + - (['--no-tags'] if skip_tags else []), - env=env, - echo=verbose) + Git.run( + args.source_root, + ["submodule", "add", remote, repo_name] + + (["--no-tags"] if skip_tags else []), + env=env, + echo=verbose, + ) else: - Git.run(args.source_root, ['clone', '--recursive', remote, repo_name] + - (['--no-tags'] if skip_tags else []), - env=env, - echo=verbose) + Git.run( + args.source_root, + ["clone", "--recursive", remote, repo_name] + + (["--no-tags"] if skip_tags else []), + env=env, + echo=verbose, + ) repo_path = os.path.join(args.source_root, repo_name) if pool_args.scheme_name: @@ -462,7 +517,7 @@ def obtain_all_additional_swift_sources( ): skipped_repositories = [] pool_args = [] - for repo_name, repo_info in config['repos'].items(): + for repo_name, repo_info in config["repos"].items(): repo_path = os.path.join(args.source_root, repo_name) if repo_name in skip_repository_list: skipped_repositories.append(SkippedReason(repo_name, "requested by user")) @@ -470,7 +525,9 @@ def obtain_all_additional_swift_sources( if args.use_submodules: repo_exists = False - submodules_status, _, _ = Git.run(repo_path, ['submodule', 'status'], fatal=True) + submodules_status, _, _ = Git.run( + repo_path, ["submodule", "status"], fatal=True + ) if submodules_status: for line in submodules_status.splitlines(): if line[0].endswith(repo_name): @@ -481,38 +538,40 @@ def obtain_all_additional_swift_sources( repo_exists = os.path.isdir(os.path.join(repo_path, ".git")) if repo_exists: - skipped_repositories.append(SkippedReason(repo_name, "directory already exists")) + skipped_repositories.append( + SkippedReason(repo_name, "directory already exists") + ) continue # If we have a url override, use that url instead of # interpolating. - remote_repo_info = repo_info['remote'] - if 'url' in remote_repo_info: - remote = remote_repo_info['url'] + remote_repo_info = repo_info["remote"] + if "url" in remote_repo_info: + remote = remote_repo_info["url"] else: - remote_repo_id = remote_repo_info['id'] - if args.clone_with_ssh is True or 'https-clone-pattern' not in config: - remote = config['ssh-clone-pattern'] % remote_repo_id + remote_repo_id = remote_repo_info["id"] + if args.clone_with_ssh is True or "https-clone-pattern" not in config: + remote = config["ssh-clone-pattern"] % remote_repo_id else: - remote = config['https-clone-pattern'] % remote_repo_id + remote = config["https-clone-pattern"] % remote_repo_id repo_branch: Optional[str] = None repo_not_in_scheme = False if scheme_name: - for v in config['branch-schemes'].values(): - if scheme_name not in v['aliases']: + for v in config["branch-schemes"].values(): + if scheme_name not in v["aliases"]: continue # If repo is not specified in the scheme, skip cloning it. - if repo_name not in v['repos']: + if repo_name not in v["repos"]: repo_not_in_scheme = True continue - repo_branch = v['repos'][repo_name] + repo_branch = v["repos"][repo_name] break else: repo_branch = scheme_name if repo_not_in_scheme: continue - + if repo_branch is None: raise RuntimeError("repo_branch is None") @@ -542,9 +601,12 @@ def obtain_all_additional_swift_sources( return [], None _move_llvm_project_to_first_index(pool_args) - return skipped_repositories, ParallelRunner( - obtain_additional_swift_sources, pool_args, args.n_processes - ).run() + return ( + skipped_repositories, + ParallelRunner( + obtain_additional_swift_sources, pool_args, args.n_processes + ).run(), + ) def dump_repo_hashes( @@ -556,19 +618,19 @@ def dump_repo_hashes( hashes. """ new_config = {} - config_copy_keys = ['ssh-clone-pattern', 'https-clone-pattern', 'repos'] + config_copy_keys = ["ssh-clone-pattern", "https-clone-pattern", "repos"] for config_copy_key in config_copy_keys: new_config[config_copy_key] = config[config_copy_key] repos = {} repos = repo_hashes(args, config) - branch_scheme = {'aliases': [branch_scheme_name], 'repos': repos} - new_config['branch-schemes'] = {branch_scheme_name: branch_scheme} + branch_scheme = {"aliases": [branch_scheme_name], "repos": repos} + new_config["branch-schemes"] = {branch_scheme_name: branch_scheme} json.dump(new_config, sys.stdout, indent=4) def repo_hashes(args: CliArguments, config: Dict[str, Any]) -> Dict[str, str]: repos = {} - for repo_name, _ in sorted(config['repos'].items(), key=lambda x: x[0]): + for repo_name, _ in sorted(config["repos"].items(), key=lambda x: x[0]): repo_path = os.path.join(args.source_root, repo_name) if os.path.exists(repo_path): h, _, _ = Git.run(repo_path, ["rev-parse", "HEAD"], fatal=True) @@ -580,12 +642,13 @@ def repo_hashes(args: CliArguments, config: Dict[str, Any]) -> Dict[str, str]: def print_repo_hashes(args: CliArguments, config: Dict[str, Any]): repos = repo_hashes(args, config) - for repo_name, repo_hash in sorted(repos.items(), - key=lambda x: x[0]): + for repo_name, repo_hash in sorted(repos.items(), key=lambda x: x[0]): print("{:<35}: {:<35}".format(repo_name, repo_hash)) -def merge_no_duplicates(a: Dict[Hashable, Any], b: Dict[Hashable, Any]) -> Dict[Hashable, Any]: +def merge_no_duplicates( + a: Dict[Hashable, Any], b: Dict[Hashable, Any] +) -> Dict[Hashable, Any]: result = {**a} for key, value in b.items(): if key in a: @@ -621,27 +684,31 @@ def merge_config(config: Dict[str, Any], new_config: Dict[str, Any]) -> Dict[str def validate_config(config: Dict[str, Any]): # Make sure that our branch-names are unique. - scheme_names = config['branch-schemes'].keys() + scheme_names = config["branch-schemes"].keys() if len(scheme_names) != len(set(scheme_names)): - raise RuntimeError('Configuration file has duplicate schemes?!') + raise RuntimeError("Configuration file has duplicate schemes?!") # Ensure the branch-scheme name is also an alias # This guarantees sensible behavior of update_repository_to_scheme when # the branch-scheme is passed as the scheme name - for scheme_name in config['branch-schemes'].keys(): - if scheme_name not in config['branch-schemes'][scheme_name]['aliases']: - raise RuntimeError('branch-scheme name: "{0}" must be an alias ' - 'too.'.format(scheme_name)) + for scheme_name in config["branch-schemes"].keys(): + if scheme_name not in config["branch-schemes"][scheme_name]["aliases"]: + raise RuntimeError( + 'branch-scheme name: "{0}" must be an alias ' "too.".format(scheme_name) + ) # Then make sure the alias names used by our branches are unique. seen: Dict[str, Any] = dict() - for (scheme_name, scheme) in config['branch-schemes'].items(): - aliases = scheme['aliases'] + for scheme_name, scheme in config["branch-schemes"].items(): + aliases = scheme["aliases"] for alias in aliases: if alias in seen: - raise RuntimeError('Configuration file defines the alias {0} ' - 'in both the {1} scheme and the {2} scheme?!' - .format(alias, seen[alias], scheme_name)) + raise RuntimeError( + "Configuration file defines the alias {0} " + "in both the {1} scheme and the {2} scheme?!".format( + alias, seen[alias], scheme_name + ) + ) else: seen[alias] = scheme_name @@ -657,7 +724,7 @@ def full_target_name(repo_path: str, repository: str, target: str) -> str: name = "%s/%s" % (repository, target) return name - raise RuntimeError('Cannot determine if %s is a branch or a tag' % target) + raise RuntimeError("Cannot determine if %s is a branch or a tag" % target) def skip_list_for_platform(config: Dict[str, Any], all_repos: bool) -> List[str]: @@ -681,9 +748,9 @@ def skip_list_for_platform(config: Dict[str, Any], all_repos: bool) -> List[str] skip_list = [] platform_name = platform.system() - for repo_name, repo_info in config['repos'].items(): - if 'platforms' in repo_info: - if platform_name not in repo_info['platforms']: + for repo_name, repo_info in config["repos"].items(): + if "platforms" in repo_info: + if platform_name not in repo_info["platforms"]: print("Skipping", repo_name, "on", platform_name) skip_list.append(repo_name) else: @@ -698,20 +765,25 @@ def main() -> int: if not args.scheme: if args.reset_to_remote: - print("update-checkout usage error: --reset-to-remote must " - "specify --scheme=foo") + print( + "update-checkout usage error: --reset-to-remote must " + "specify --scheme=foo" + ) sys.exit(1) if args.match_timestamp: # without a scheme, we won't be able match timestamps forward in # time, which is an annoying footgun for bisection etc. - print("update-checkout usage error: --match-timestamp must " - "specify --scheme=foo") + print( + "update-checkout usage error: --match-timestamp must " + "specify --scheme=foo" + ) sys.exit(1) # Set the default config path if none are specified if not args.configs: - default_path = os.path.join(SCRIPT_DIR, os.pardir, - "update-checkout-config.json") + default_path = os.path.join( + SCRIPT_DIR, os.pardir, "update-checkout-config.json" + ) args.configs.append(default_path) config: Dict[str, Any] = {} for config_path in args.configs: @@ -721,20 +793,22 @@ def main() -> int: cross_repos_pr: Dict[str, str] = {} if args.github_comment: - regex_pr = r'(apple/[-a-zA-Z0-9_]+/pull/\d+'\ - r'|apple/[-a-zA-Z0-9_]+#\d+'\ - r'|swiftlang/[-a-zA-Z0-9_]+/pull/\d+'\ - r'|swiftlang/[-a-zA-Z0-9_]+#\d+)' + regex_pr = ( + r"(apple/[-a-zA-Z0-9_]+/pull/\d+" + r"|apple/[-a-zA-Z0-9_]+#\d+" + r"|swiftlang/[-a-zA-Z0-9_]+/pull/\d+" + r"|swiftlang/[-a-zA-Z0-9_]+#\d+)" + ) repos_with_pr = re.findall(regex_pr, args.github_comment) print("Found related pull requests:", str(repos_with_pr)) - repos_with_pr = [pr.replace('/pull/', '#') for pr in repos_with_pr] - cross_repos_pr = dict(pr.split('#') for pr in repos_with_pr) + repos_with_pr = [pr.replace("/pull/", "#") for pr in repos_with_pr] + cross_repos_pr = dict(pr.split("#") for pr in repos_with_pr) # If branch is None, default to using the default branch alias # specified by our configuration file. scheme_name = args.scheme if scheme_name is None: - scheme_name = config['default-branch-scheme'] + scheme_name = config["default-branch-scheme"] scheme_map = get_scheme_map(config, scheme_name) @@ -748,13 +822,12 @@ def main() -> int: ) SkippedReason.print_skipped_repositories(skipped_repositories, "clone") - swift_repo_path = os.path.join(args.source_root, 'swift') - if 'swift' not in skip_repo_list and os.path.exists(swift_repo_path): + swift_repo_path = os.path.join(args.source_root, "swift") + if "swift" not in skip_repo_list and os.path.exists(swift_repo_path): # Check if `swift` repo itself needs to switch to a cross-repo branch. - branch_name, cross_repo = get_branch_for_repo(swift_repo_path, config, 'swift', - scheme_name, - scheme_map, - cross_repos_pr) + branch_name, cross_repo = get_branch_for_repo( + swift_repo_path, config, "swift", scheme_name, scheme_map, cross_repos_pr + ) if cross_repo: Git.run( @@ -779,14 +852,19 @@ def main() -> int: # Quick check whether somebody is calling update in an empty directory directory_contents = os.listdir(args.source_root) - if not ('cmark' in directory_contents or - 'llvm' in directory_contents or - 'clang' in directory_contents): - print("You don't have all swift sources. " - "Call this script with --clone to get them.") - - skipped_repositories, update_results = update_all_repositories(args, config, scheme_name, - scheme_map, cross_repos_pr) + if not ( + "cmark" in directory_contents + or "llvm" in directory_contents + or "clang" in directory_contents + ): + print( + "You don't have all swift sources. " + "Call this script with --clone to get them." + ) + + skipped_repositories, update_results = update_all_repositories( + args, config, scheme_name, scheme_map, cross_repos_pr + ) SkippedReason.print_skipped_repositories(skipped_repositories, "update") fail_count = 0