Use tmp deploy key during migration

.github/workflows/collection-migration-tests.yml

@@ -38,18 +38,6 @@ jobs:
       run: |
         git config --global user.email "ansible_migration@example.com"
         git config --global user.name "Poor B"
-    # https://www.webfactory.de/blog/use-ssh-key-for-private-repositories-in-github-actions
-    - name: Setup SSH Keys and known_hosts
-      if: >-
-        github.event_name != 'pull_request'
-        && github.repository == 'ansible-community/collection_migration'
-      env:
-        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
-      run: |
-        mkdir -p ~/.ssh
-        ssh-keyscan github.com >> ~/.ssh/known_hosts
-        ssh-agent -a $SSH_AUTH_SOCK > /dev/null
-        ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY_MINIMAL }}"
     - name: >-
         Run migration scenario ${{ matrix.migration-scenario }}
         and auto-publish the migrated repos to GitHub repos
@@ -59,13 +47,14 @@ jobs:
       env:
         GITHUB_APP_IDENTIFIER: 41435
         GITHUB_PRIVATE_KEY: ${{ secrets.GITHUB_PRIVATE_KEY }}
-        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
       run: >-
         python -m
         migrate
         -m
         -s "scenarios/${{ matrix.migration-scenario }}"
-        -M
+        --publish-to-github
+        --target-github-org ansible-collection-migration
+        --push-migrated-core
     - name: Smoke test ansible-minimal
       if: >-
         github.event_name != 'pull_request'

gh.py

@@ -1,5 +1,7 @@
 """GitHub App auth and helpers."""
 
+from __future__ import annotations
+
 import asyncio
 from dataclasses import dataclass
 import contextlib
@@ -22,6 +24,8 @@ class GitHubOrgClient:
     github_app_private_key_path: Union[pathlib.Path, str]
     github_org_name: str
 
+    deployment_rsa_pub_key: str
+
     def _read_app_id(self):
         if self.github_app_id is None:
             return int(os.environ['GITHUB_APP_IDENTIFIER'])
@@ -95,3 +99,60 @@ async def get_git_repo_token(self, repo_name):
     def get_git_repo_write_uri(self, repo_name):
         """Get a Git repo URL with embedded creds synchronously."""
         return asyncio.run(self.get_git_repo_token(repo_name))
+
+    def sync_provision_deploy_key_to(self, repo_name: str) -> int:
+        return asyncio.run(self.provision_deploy_key_to(repo_name))
+
+    async def provision_deploy_key_to(self, repo_name: str) -> int:
+        """Add deploy key to the repo."""
+        await self.create_repo_if_not_exists(repo_name)
+
+        dpl_key = self.deployment_rsa_pub_key
+        dpl_key_repr = dpl_key.split(' ')[1]
+        dpl_key_repr = '...'.join((dpl_key_repr[:16], dpl_key_repr[-16:]))
+        github_api = await self._get_github_client()
+        api_resp = await github_api.post(
+            '/repos/{owner}/{repo}/keys',
+            url_vars={
+                'owner': self.github_org_name,
+                'repo': repo_name,
+            },
+            data={
+                'title': (
+                    '[SHOULD BE AUTO-REMOVED MINUTES AFTER CREATION!] '
+                    f'Temporary key ({dpl_key_repr}) added '
+                    'by Ansible Collection Migrator'
+                ),
+                'key': dpl_key,
+                'read_only': False,
+            },
+        )
+        return api_resp['id']
+
+    def sync_drop_deploy_key_from(self, repo_name: str, key_id: int):
+        return asyncio.run(self.drop_deploy_key_from(repo_name, key_id))
+
+    async def drop_deploy_key_from(self, repo_name: str, key_id: int):
+        """Add deploy key to the repo."""
+        github_api = await self._get_github_client()
+        await github_api.delete(
+            '/repos/{owner}/{repo}/keys/{key_id}',
+            url_vars={
+                'owner': self.github_org_name,
+                'repo': repo_name,
+                'key_id': key_id,
+            },
+        )
+
+    def tmp_deployment_key_for(self, repo_name: str):
+        """Make a CM that adds and removes deployment keys."""
+        return _tmp_repo_deploy_key(self, repo_name)
+
+
+@contextlib.contextmanager
+def _tmp_repo_deploy_key(gh_api, repo_name):
+    _key_id = gh_api.sync_provision_deploy_key_to(repo_name)
+    try:
+        yield
+    finally:
+        gh_api.sync_drop_deploy_key_from(repo_name, _key_id)

migrate.py

@@ -33,6 +33,7 @@
 import redbaron
 
 from gh import GitHubOrgClient
+from rsa_utils import RSAKey
 from template_utils import render_template_into
 
 
@@ -42,7 +43,7 @@
 
 DEVEL_URL = 'https://github.com/ansible/ansible.git'
 DEVEL_BRANCH = 'devel'
-MIGRATED_DEVEL_REMOTE = 'git@github.com:ansible-collection-migration/ansible-minimal.git'
+MIGRATED_DEVEL_REPO_NAME = 'ansible-minimal'
 
 
 ALL_THE_FILES = set()
@@ -1331,7 +1332,7 @@ def add_deps_to_metadata(deps, galaxy_metadata):
         galaxy_metadata['dependencies'][dep] = '>=1.0'
 
 
-def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app_key_path):
+def publish_to_github(collections_target_dir, spec, github_api, rsa_key):
     """Push all migrated collections to their Git remotes."""
     collections_base_dir = os.path.join(collections_target_dir, 'collections')
     collections_root_dir = os.path.join(
@@ -1344,24 +1345,15 @@ def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app
         for coll in ns_val.keys()
         if not coll.startswith('_')
     )
-    github_api = GitHubOrgClient(gh_app_id, gh_app_key_path, gh_org)
-    for collection_dir, repo_name in collection_paths_except_core:
-        git_repo_url = read_yaml_file(
-            os.path.join(collection_dir, 'galaxy.yml'),
-        )['repository']
-        with contextlib.suppress(LookupError):
-            git_repo_url = github_api.get_git_repo_write_uri(repo_name)
-        logger.debug(
-            'Using %s...%s Git URL for push',
-            git_repo_url[:5], git_repo_url[-5:],
-        )
-        logger.info(
-            'Rebasing the migrated collection on top of the Git remote',
-        )
-        # Putting our newly generated stuff on top of what's on remote:
-        # Ref: https://demisx.github.io/git/rebase/2015/07/02/git-rebase-keep-my-branch-changes.html
-        subprocess.check_call(
-            (
+    logger.debug('Using SSH key %s...', rsa_key.public_openssh)
+    with rsa_key.ssh_agent as ssh_agent:
+        for collection_dir, repo_name in collection_paths_except_core:
+            git_repo_url = read_yaml_file(
+                os.path.join(collection_dir, 'galaxy.yml'),
+            )['repository']
+            # Putting our newly generated stuff on top of what's on remote:
+            # Ref: https://demisx.github.io/git/rebase/2015/07/02/git-rebase-keep-my-branch-changes.html
+            git_pull_rebase_cmd = (
                 'git', 'pull',
                 '--allow-unrelated-histories',
                 '--rebase',
@@ -1372,36 +1364,76 @@ def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app
                 # * https://dev.to/willamesoares/git-ours-or-theirs-part-2-d0o
                 '--strategy-option', 'theirs',
                 git_repo_url,
-                'master'
-            ),
-            cwd=collection_dir,
-        )
-        logger.info('Pushing the migrated collection to the Git remote')
-        subprocess.check_call(
-            ('git', 'push', '--force', git_repo_url, 'HEAD:master'),
-            cwd=collection_dir,
-        )
-        logger.info(
-            'The migrated collection has been successfully published to '
-            '`https://github.com/%s/%s.git`...',
-            gh_org,
-            repo_name,
-        )
+                'master',
+            )
+            # with contextlib.suppress(LookupError):
+            #     git_repo_url = github_api.get_git_repo_write_uri(repo_name)
+            git_repo_url_repr = '...'.join((
+                git_repo_url[:5], git_repo_url[-5:],
+            )) if not git_repo_url.startswith('git@') else git_repo_url
+            logger.debug(
+                'Using %s Git URL for push',
+                git_repo_url_repr,
+            )
+            logger.info(
+                'Rebasing the migrated collection in '
+                'repo %s on top of the Git remote',
+                repo_name,
+            )
+            logger.debug('Invoking `%s`...', ' '.join(git_pull_rebase_cmd))
+            with github_api.tmp_deployment_key_for(repo_name):
+                try:
+                    ssh_agent.run(
+                        git_pull_rebase_cmd,
+                        cwd=collection_dir,
+                        check=True,
+                        text=True,
+                        stdin=subprocess.DEVNULL,
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.PIPE,
+                    )
+                except subprocess.CalledProcessError as proc_err:
+                    # Ref: https://github.com/git/git/commit/b97e187364990fb8410355ff8b4365d0e37bbbbe
+                    acceptable_endings = {
+                        'error: nothing to do',
+                        "fatal: couldn't find remote ref master",
+                    }
+                    has_acceptable_stderr = proc_err.stderr[:-1].endswith
+                    if not any(
+                        has_acceptable_stderr(o)
+                        for o in acceptable_endings
+                    ):
+                        raise
+                logger.info('Pushing the migrated collection to the Git remote')
+                ssh_agent.check_call(
+                    ('git', 'push', '--force', git_repo_url, 'HEAD:master'),
+                    cwd=collection_dir,
+                )
+                logger.info(
+                    'The migrated collection has been successfully published to '
+                    '`https://github.com/%s/%s.git`...',
+                    github_api.github_org_name,
+                    repo_name,
+                )
 
 
-def push_migrated_core(releases_dir):
+def push_migrated_core(releases_dir, github_api, rsa_key):
     devel_path = os.path.join(releases_dir, f'{DEVEL_BRANCH}.git')
 
-    subprocess.check_call(
-        ('git', 'remote', 'add', 'migrated_core', MIGRATED_DEVEL_REMOTE),
-        cwd=devel_path,
+    migrated_devel_remote = (
+        f'git@github.com:{github_api.github_org_name}/'
+        f'{MIGRATED_DEVEL_REPO_NAME}.git'
     )
 
-    # NOTE: assumes the repo is not used and/or is locked while migration is running
-    subprocess.check_call(
-        ('git', 'push', '--force', 'migrated_core', DEVEL_BRANCH),
-        cwd=devel_path,
-    )
+    logger.debug('Using SSH key %s...', rsa_key.public_openssh)
+    with rsa_key.ssh_agent as ssh_agent, github_api.tmp_deployment_key_for(
+            MIGRATED_DEVEL_REPO_NAME,
+    ):
+        # NOTE: assumes the repo is not used and/or is locked while migration is running
+        ssh_agent.check_call(
+            ('git', 'push', '--force', migrated_devel_remote, DEVEL_BRANCH),
+            cwd=devel_path,
+        )
 
 
 def assert_migrating_git_tracked_resources(
@@ -1991,16 +2023,24 @@ def main():
     # doeet
     assemble_collections(devel_path, spec, args, args.target_github_org)
 
+    tmp_rsa_key = None
+    github_api = None
+    if args.publish_to_github or args.push_migrated_core:
+        tmp_rsa_key = RSAKey()
+        gh_api = GitHubOrgClient(
+            args.github_app_id, args.github_app_key_path,
+            args.target_github_org,
+            deployment_rsa_pub_key=tmp_rsa_key.public_openssh,
+        )
+
     if args.publish_to_github:
         publish_to_github(
             args.vardir, spec,
-            gh_org=args.target_github_org,
-            gh_app_id=args.github_app_id,
-            gh_app_key_path=args.github_app_key_path,
+            gh_api, tmp_rsa_key,
         )
 
     if args.push_migrated_core:
-        push_migrated_core(releases_dir)
+        push_migrated_core(releases_dir, gh_api, tmp_rsa_key)
 
     global core
     print('======= Assumed stayed in core =======\n')

requirements.in

@@ -1,4 +1,5 @@
 ansible
+cryptography  # dependency of octomachinery but imported too
 gidgethub  # dependency of octomachinery but imported too
 Jinja2
 logzero