From affccfb821e7daf3e27183c3136ddbb7da00da14 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 7 Jan 2020 01:10:45 +0100 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20Add=20in-mem=20RSA=20key=20supp?= =?UTF-8?q?ort=20with=20ssh-agent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change encapsulates RSA public-private key pair generation using cryptography under the hood. It also implments a context manager interface that manages ssh-agent and wraps a few subprocess function calls with a pre-populated env bindings to the said ssh-agent instance. --- requirements.in | 1 + rsa_utils.py | 152 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 rsa_utils.py diff --git a/requirements.in b/requirements.in index a5f28988..e75d9ff7 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,5 @@ ansible +cryptography # dependency of octomachinery but imported too gidgethub # dependency of octomachinery but imported too Jinja2 logzero diff --git a/rsa_utils.py b/rsa_utils.py new file mode 100644 index 00000000..39064fd4 --- /dev/null +++ b/rsa_utils.py @@ -0,0 +1,152 @@ +"""In-memory RSA key generation and management utils.""" +from __future__ import annotations + +import contextlib +import functools +import os +import subprocess + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.rsa import generate_private_key +from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PublicFormat, + PrivateFormat, +) + + +class RSAKey: + """In-memory RSA key wrapper.""" + + def __init__(self): + _rsa_key_obj = generate_private_key( + public_exponent=65537, + key_size=4096, + backend=default_backend(), + ) + + _rsa_pub_key_obj = _rsa_key_obj.public_key() + self._public_repr = _rsa_pub_key_obj.public_bytes( + encoding=Encoding.PEM, + format=PublicFormat.PKCS1, + ).decode() + self._public_repr_openssh = _rsa_pub_key_obj.public_bytes( + encoding=Encoding.OpenSSH, + format=PublicFormat.OpenSSH, + ).decode() + + _private_rsa_key_repr = _rsa_key_obj.private_bytes( + encoding=Encoding.PEM, + format=PrivateFormat.TraditionalOpenSSL, # A.K.A. PKCS#1 + encryption_algorithm=NoEncryption(), + ) + self._ssh_agent_cm = SSHAgent(_private_rsa_key_repr) + + @property + def ssh_agent(self) -> SSHAgent: + """SSH agent CM.""" + return self._ssh_agent_cm + + @property + def public(self) -> str: + """String PKCS#1-formatted representation of the public key.""" + return self._public_repr + + @property + def public_openssh(self) -> str: + """String OpenSSH-formatted representation of the public key.""" + return self._public_repr_openssh + + +class SSHAgent: + """SSH agent lifetime manager. + + Only usable as a CM. Only holds one RSA key in memory. + """ + + def __init__(self, ssh_key: bytes): + self._ssh_key = ssh_key + self._ssh_agent_proc = None + self._ssh_agent_socket = None + + def __enter__(self) -> _SubprocessSSHAgentProxy: + ssh_agent_cmd = ( + 'ssh-agent', # man 1 ssh-agent + '-s', # generate Bourne shell commands on stdout + '-D', # foreground mode + ) + ssh_add_cmd = 'ssh-add', '-' + + self._ssh_agent_proc = subprocess.Popen( + ssh_agent_cmd, + stdout=subprocess.PIPE, # we need to parse the socket path + text=True, # auto-decode the text from bytes + ) + self._ssh_agent_socket = ( + self._ssh_agent_proc. + stdout.readline(). + partition('; ')[0]. + partition('=')[-1] + ) + + subprocess_proxy = _SubprocessSSHAgentProxy(self._ssh_agent_socket) + subprocess_proxy.check_output( + ssh_add_cmd, + input=self._ssh_key, + stderr=subprocess.DEVNULL, + ) + + return subprocess_proxy + + def __exit__(self, exc_type, exc_val, exc_tb): + ssh_agent_proc = self._ssh_agent_proc + + self._ssh_agent_socket = None + self._ssh_agent_proc = None + + with contextlib.suppress(IOError, OSError): + ssh_agent_proc.terminate() + + return False + + +def pre_populate_env_kwarg(meth): + """Pre-populated env arg in decorated methods.""" + @functools.wraps(meth) + def method_wrapper(self, *args, **kwargs): + if 'env' not in kwargs: + kwargs['env'] = os.environ.copy() + kwargs['env']['GIT_SSH_COMMAND'] = ( + 'ssh -2 ' + '-F /dev/null ' + '-o PreferredAuthentications=publickey ' + '-o IdentityFile=/dev/null ' + f'-o IdentityAgent="{self._sock}"' + ) + kwargs['env']['SSH_AUTH_SOCK'] = self._sock + return meth(self, *args, **kwargs) + return method_wrapper + + +# pylint: disable=too-few-public-methods +class _SubprocessSSHAgentProxy: + """Proxy object for calls to subprocess functions.""" + + def __init__(self, sock): + self._sock = sock + + @pre_populate_env_kwarg + def check_call(self, *args, **kwargs): + """Populate the SSH agent sock into the check_call env.""" + return subprocess.check_call(*args, **kwargs) + + @pre_populate_env_kwarg + def check_output(self, *args, **kwargs): + """Populate the SSH agent sock into the check_output env.""" + return subprocess.check_output(*args, **kwargs) + + @pre_populate_env_kwarg + def run(self, *args, **kwargs): + """Populate the SSH agent sock into the run env.""" + return subprocess.run(*args, **kwargs) From 13f234ddc19696d5e220e9f6510e010c4d8d4fd2 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 7 Jan 2020 01:14:39 +0100 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8Make=20a=20tmp=20dpl=20key=20provi?= =?UTF-8?q?sioner=20@=20GH=20API=20wrapper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It accepts a pre-existing RSA private key string and adds it to the given repo on enter. On exit, it drops this key from that repo. --- gh.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/gh.py b/gh.py index 774efad2..2e894ce6 100644 --- a/gh.py +++ b/gh.py @@ -1,5 +1,7 @@ """GitHub App auth and helpers.""" +from __future__ import annotations + import asyncio from dataclasses import dataclass import contextlib @@ -22,6 +24,8 @@ class GitHubOrgClient: github_app_private_key_path: Union[pathlib.Path, str] github_org_name: str + deployment_rsa_pub_key: str + def _read_app_id(self): if self.github_app_id is None: return int(os.environ['GITHUB_APP_IDENTIFIER']) @@ -95,3 +99,60 @@ async def get_git_repo_token(self, repo_name): def get_git_repo_write_uri(self, repo_name): """Get a Git repo URL with embedded creds synchronously.""" return asyncio.run(self.get_git_repo_token(repo_name)) + + def sync_provision_deploy_key_to(self, repo_name: str) -> int: + return asyncio.run(self.provision_deploy_key_to(repo_name)) + + async def provision_deploy_key_to(self, repo_name: str) -> int: + """Add deploy key to the repo.""" + await self.create_repo_if_not_exists(repo_name) + + dpl_key = self.deployment_rsa_pub_key + dpl_key_repr = dpl_key.split(' ')[1] + dpl_key_repr = '...'.join((dpl_key_repr[:16], dpl_key_repr[-16:])) + github_api = await self._get_github_client() + api_resp = await github_api.post( + '/repos/{owner}/{repo}/keys', + url_vars={ + 'owner': self.github_org_name, + 'repo': repo_name, + }, + data={ + 'title': ( + '[SHOULD BE AUTO-REMOVED MINUTES AFTER CREATION!] ' + f'Temporary key ({dpl_key_repr}) added ' + 'by Ansible Collection Migrator' + ), + 'key': dpl_key, + 'read_only': False, + }, + ) + return api_resp['id'] + + def sync_drop_deploy_key_from(self, repo_name: str, key_id: int): + return asyncio.run(self.drop_deploy_key_from(repo_name, key_id)) + + async def drop_deploy_key_from(self, repo_name: str, key_id: int): + """Add deploy key to the repo.""" + github_api = await self._get_github_client() + await github_api.delete( + '/repos/{owner}/{repo}/keys/{key_id}', + url_vars={ + 'owner': self.github_org_name, + 'repo': repo_name, + 'key_id': key_id, + }, + ) + + def tmp_deployment_key_for(self, repo_name: str): + """Make a CM that adds and removes deployment keys.""" + return _tmp_repo_deploy_key(self, repo_name) + + +@contextlib.contextmanager +def _tmp_repo_deploy_key(gh_api, repo_name): + _key_id = gh_api.sync_provision_deploy_key_to(repo_name) + try: + yield + finally: + gh_api.sync_drop_deploy_key_from(repo_name, _key_id) From 1d268ce077b580ea616a93d4edb00279f9943547 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 7 Jan 2020 01:17:07 +0100 Subject: [PATCH 3/5] =?UTF-8?q?=E2=99=BB=20Switch=20to=20using=20temporary?= =?UTF-8?q?=20deploy=20keys=20in=20Git?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- migrate.py | 99 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/migrate.py b/migrate.py index 20a56569..720de8f5 100755 --- a/migrate.py +++ b/migrate.py @@ -33,6 +33,7 @@ import redbaron from gh import GitHubOrgClient +from rsa_utils import RSAKey from template_utils import render_template_into @@ -1344,24 +1345,20 @@ def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app for coll in ns_val.keys() if not coll.startswith('_') ) - github_api = GitHubOrgClient(gh_app_id, gh_app_key_path, gh_org) - for collection_dir, repo_name in collection_paths_except_core: - git_repo_url = read_yaml_file( - os.path.join(collection_dir, 'galaxy.yml'), - )['repository'] - with contextlib.suppress(LookupError): - git_repo_url = github_api.get_git_repo_write_uri(repo_name) - logger.debug( - 'Using %s...%s Git URL for push', - git_repo_url[:5], git_repo_url[-5:], - ) - logger.info( - 'Rebasing the migrated collection on top of the Git remote', - ) - # Putting our newly generated stuff on top of what's on remote: - # Ref: https://demisx.github.io/git/rebase/2015/07/02/git-rebase-keep-my-branch-changes.html - subprocess.check_call( - ( + tmp_rsa_key = RSAKey() + github_api = GitHubOrgClient( + gh_app_id, gh_app_key_path, gh_org, + deployment_rsa_pub_key=tmp_rsa_key.public_openssh, + ) + logger.debug('Using SSH key %s...', tmp_rsa_key.public_openssh) + with tmp_rsa_key.ssh_agent as ssh_agent: + for collection_dir, repo_name in collection_paths_except_core: + git_repo_url = read_yaml_file( + os.path.join(collection_dir, 'galaxy.yml'), + )['repository'] + # Putting our newly generated stuff on top of what's on remote: + # Ref: https://demisx.github.io/git/rebase/2015/07/02/git-rebase-keep-my-branch-changes.html + git_pull_rebase_cmd = ( 'git', 'pull', '--allow-unrelated-histories', '--rebase', @@ -1372,21 +1369,57 @@ def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app # * https://dev.to/willamesoares/git-ours-or-theirs-part-2-d0o '--strategy-option', 'theirs', git_repo_url, - 'master' - ), - cwd=collection_dir, - ) - logger.info('Pushing the migrated collection to the Git remote') - subprocess.check_call( - ('git', 'push', '--force', git_repo_url, 'HEAD:master'), - cwd=collection_dir, - ) - logger.info( - 'The migrated collection has been successfully published to ' - '`https://github.com/%s/%s.git`...', - gh_org, - repo_name, - ) + 'master', + ) + # with contextlib.suppress(LookupError): + # git_repo_url = github_api.get_git_repo_write_uri(repo_name) + git_repo_url_repr = '...'.join(( + git_repo_url[:5], git_repo_url[-5:], + )) if not git_repo_url.startswith('git@') else git_repo_url + logger.debug( + 'Using %s Git URL for push', + git_repo_url_repr, + ) + logger.info( + 'Rebasing the migrated collection in ' + 'repo %s on top of the Git remote', + repo_name, + ) + logger.debug('Invoking `%s`...', ' '.join(git_pull_rebase_cmd)) + with github_api.tmp_deployment_key_for(repo_name): + try: + ssh_agent.run( + git_pull_rebase_cmd, + cwd=collection_dir, + check=True, + text=True, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as proc_err: + # Ref: https://github.com/git/git/commit/b97e187364990fb8410355ff8b4365d0e37bbbbe + acceptable_endings = { + 'error: nothing to do', + "fatal: couldn't find remote ref master", + } + has_acceptable_stderr = proc_err.stderr[:-1].endswith + if not any( + has_acceptable_stderr(o) + for o in acceptable_endings + ): + raise + logger.info('Pushing the migrated collection to the Git remote') + ssh_agent.check_call( + ('git', 'push', '--force', git_repo_url, 'HEAD:master'), + cwd=collection_dir, + ) + logger.info( + 'The migrated collection has been successfully published to ' + '`https://github.com/%s/%s.git`...', + gh_org, + repo_name, + ) def push_migrated_core(releases_dir): From b8670883c01c8cca6cf8830be25f5b450934eeda Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 7 Jan 2020 01:47:41 +0100 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=9A=91=20Restore=20publishing=20migra?= =?UTF-8?q?ted=20repos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ref #176 --- .github/workflows/collection-migration-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/collection-migration-tests.yml b/.github/workflows/collection-migration-tests.yml index 86fc9be6..5056e452 100644 --- a/.github/workflows/collection-migration-tests.yml +++ b/.github/workflows/collection-migration-tests.yml @@ -65,6 +65,8 @@ jobs: migrate -m -s "scenarios/${{ matrix.migration-scenario }}" + --publish-to-github + --target-github-org ansible-collection-migration -M - name: Smoke test ansible-minimal if: >- From b6a52c91f9b217fef4e4a6685249e72251312cb3 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 7 Jan 2020 01:55:00 +0100 Subject: [PATCH 5/5] =?UTF-8?q?=E2=99=BB=F0=9F=8E=A8Switch=20to=20pushing?= =?UTF-8?q?=20migrated=20core=20via=20GitHub=20App?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #206 --- .../workflows/collection-migration-tests.yml | 15 +----- migrate.py | 53 +++++++++++-------- 2 files changed, 31 insertions(+), 37 deletions(-) diff --git a/.github/workflows/collection-migration-tests.yml b/.github/workflows/collection-migration-tests.yml index 5056e452..368a122e 100644 --- a/.github/workflows/collection-migration-tests.yml +++ b/.github/workflows/collection-migration-tests.yml @@ -38,18 +38,6 @@ jobs: run: | git config --global user.email "ansible_migration@example.com" git config --global user.name "Poor B" - # https://www.webfactory.de/blog/use-ssh-key-for-private-repositories-in-github-actions - - name: Setup SSH Keys and known_hosts - if: >- - github.event_name != 'pull_request' - && github.repository == 'ansible-community/collection_migration' - env: - SSH_AUTH_SOCK: /tmp/ssh_agent.sock - run: | - mkdir -p ~/.ssh - ssh-keyscan github.com >> ~/.ssh/known_hosts - ssh-agent -a $SSH_AUTH_SOCK > /dev/null - ssh-add - <<< "${{ secrets.SSH_PRIVATE_KEY_MINIMAL }}" - name: >- Run migration scenario ${{ matrix.migration-scenario }} and auto-publish the migrated repos to GitHub repos @@ -59,7 +47,6 @@ jobs: env: GITHUB_APP_IDENTIFIER: 41435 GITHUB_PRIVATE_KEY: ${{ secrets.GITHUB_PRIVATE_KEY }} - SSH_AUTH_SOCK: /tmp/ssh_agent.sock run: >- python -m migrate @@ -67,7 +54,7 @@ jobs: -s "scenarios/${{ matrix.migration-scenario }}" --publish-to-github --target-github-org ansible-collection-migration - -M + --push-migrated-core - name: Smoke test ansible-minimal if: >- github.event_name != 'pull_request' diff --git a/migrate.py b/migrate.py index 720de8f5..64a4c479 100755 --- a/migrate.py +++ b/migrate.py @@ -43,7 +43,7 @@ DEVEL_URL = 'https://github.com/ansible/ansible.git' DEVEL_BRANCH = 'devel' -MIGRATED_DEVEL_REMOTE = 'git@github.com:ansible-collection-migration/ansible-minimal.git' +MIGRATED_DEVEL_REPO_NAME = 'ansible-minimal' ALL_THE_FILES = set() @@ -1332,7 +1332,7 @@ def add_deps_to_metadata(deps, galaxy_metadata): galaxy_metadata['dependencies'][dep] = '>=1.0' -def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app_key_path): +def publish_to_github(collections_target_dir, spec, github_api, rsa_key): """Push all migrated collections to their Git remotes.""" collections_base_dir = os.path.join(collections_target_dir, 'collections') collections_root_dir = os.path.join( @@ -1345,13 +1345,8 @@ def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app for coll in ns_val.keys() if not coll.startswith('_') ) - tmp_rsa_key = RSAKey() - github_api = GitHubOrgClient( - gh_app_id, gh_app_key_path, gh_org, - deployment_rsa_pub_key=tmp_rsa_key.public_openssh, - ) - logger.debug('Using SSH key %s...', tmp_rsa_key.public_openssh) - with tmp_rsa_key.ssh_agent as ssh_agent: + logger.debug('Using SSH key %s...', rsa_key.public_openssh) + with rsa_key.ssh_agent as ssh_agent: for collection_dir, repo_name in collection_paths_except_core: git_repo_url = read_yaml_file( os.path.join(collection_dir, 'galaxy.yml'), @@ -1417,24 +1412,28 @@ def publish_to_github(collections_target_dir, spec, *, gh_org, gh_app_id, gh_app logger.info( 'The migrated collection has been successfully published to ' '`https://github.com/%s/%s.git`...', - gh_org, + github_api.github_org_name, repo_name, ) -def push_migrated_core(releases_dir): +def push_migrated_core(releases_dir, github_api, rsa_key): devel_path = os.path.join(releases_dir, f'{DEVEL_BRANCH}.git') - subprocess.check_call( - ('git', 'remote', 'add', 'migrated_core', MIGRATED_DEVEL_REMOTE), - cwd=devel_path, + migrated_devel_remote = ( + f'git@github.com:{github_api.github_org_name}/' + f'{MIGRATED_DEVEL_REPO_NAME}.git' ) - # NOTE: assumes the repo is not used and/or is locked while migration is running - subprocess.check_call( - ('git', 'push', '--force', 'migrated_core', DEVEL_BRANCH), - cwd=devel_path, - ) + logger.debug('Using SSH key %s...', rsa_key.public_openssh) + with rsa_key.ssh_agent as ssh_agent, github_api.tmp_deployment_key_for( + MIGRATED_DEVEL_REPO_NAME, + ): + # NOTE: assumes the repo is not used and/or is locked while migration is running + ssh_agent.check_call( + ('git', 'push', '--force', migrated_devel_remote, DEVEL_BRANCH), + cwd=devel_path, + ) def assert_migrating_git_tracked_resources( @@ -2024,16 +2023,24 @@ def main(): # doeet assemble_collections(devel_path, spec, args, args.target_github_org) + tmp_rsa_key = None + github_api = None + if args.publish_to_github or args.push_migrated_core: + tmp_rsa_key = RSAKey() + gh_api = GitHubOrgClient( + args.github_app_id, args.github_app_key_path, + args.target_github_org, + deployment_rsa_pub_key=tmp_rsa_key.public_openssh, + ) + if args.publish_to_github: publish_to_github( args.vardir, spec, - gh_org=args.target_github_org, - gh_app_id=args.github_app_id, - gh_app_key_path=args.github_app_key_path, + gh_api, tmp_rsa_key, ) if args.push_migrated_core: - push_migrated_core(releases_dir) + push_migrated_core(releases_dir, gh_api, tmp_rsa_key) global core print('======= Assumed stayed in core =======\n')