From 532751627a347637814e8a7f626dad487a1b6053 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 10 Feb 2020 10:39:07 -0500 Subject: [PATCH 01/20] Backported changes to CONTRIBUTING.md from the development guide. --- CONTRIBUTING.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93addc2..eb00ca9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,10 +56,31 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux (or on the Mac, if you don't want to use `brew`) you can use +For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to -install the necessary tools. When you are finished you will need to -add the same two lines above to your profile. +install the necessary tools. Before running this ensure that you have +installed the prerequisites for your platform according to the +[`pyenv` wiki +page](https://github.com/pyenv/pyenv/wiki/common-build-problems). + +On WSL you should treat your platform as whatever Linux distribution +you've chosen to install. + +Once you have installed `pyenv` you will need to add the following +lines to your `.bashrc`: + +```bash +export PATH="$PATH:$HOME/.pyenv/bin" +eval "$(pyenv init -)" +eval "$(pyenv virtualenv-init -)" +``` + +If you are using a shell other than `bash` you should follow the +instructions that the `pyenv-installer` script outputs. + +You will need to reload your shell for these changes to take effect so +you can begin to use `pyenv`. For a list of Python versions that are already installed and ready to use with `pyenv`, use the command `pyenv versions`. To see a list of From f7a4166ad67d961324bc44130e092eb1ddebd320 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 11 Feb 2020 10:41:17 -0500 Subject: [PATCH 02/20] Update Python version used to 3.8 Update actions/checkout to v2 Update formatting to match downstream children --- .github/workflows/build.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6c14e6..4953f7c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,20 +10,16 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - - name: Set up Python 3.7 - uses: actions/setup-python@v1 + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 with: - python-version: 3.7 - + python-version: 3.8 - name: Cache pre-commit hooks uses: actions/cache@v1 with: path: ~/.cache/pre-commit key: "${{ runner.os }}-pre-commit-\ ${{ hashFiles('**/.pre-commit-config.yaml') }}" - - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -33,11 +29,9 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- - - name: Install dependencies run: | python -m pip install --upgrade pip pip install --upgrade -r requirements-test.txt - - name: Run pre-commit on all files run: pre-commit run --all-files From b857939b3d0c5393b8b5528a6bb9bce2ff0f736f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:01:16 -0500 Subject: [PATCH 03/20] Run pre-commit autoupdate. --- .pre-commit-config.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c8fc88a..7856658 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.4.0 + rev: v2.5.0 hooks: - id: check-executables-have-shebangs - id: check-json @@ -27,13 +27,13 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.19.0 + rev: v0.22.0 hooks: - id: markdownlint args: - --config=.mdl_config.json - repo: https://github.com/adrienverge/yamllint - rev: v1.18.0 + rev: v1.20.0 hooks: - id: yamllint - repo: https://github.com/detailyang/pre-commit-shell @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.25.1 + rev: v1.26.2 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -61,7 +61,7 @@ repos: hooks: - id: black - repo: https://github.com/asottile/seed-isort-config - rev: v1.9.3 + rev: v1.9.4 hooks: - id: seed-isort-config - repo: https://github.com/pre-commit/mirrors-isort @@ -71,7 +71,7 @@ repos: hooks: - id: isort - repo: https://github.com/ansible/ansible-lint.git - rev: v4.1.1a5 + rev: v4.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml @@ -81,7 +81,7 @@ repos: - id: terraform_fmt - id: terraform_validate_no_variables - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v1.0.0 + rev: v1.0.1 hooks: - id: docker-compose-check - repo: https://github.com/prettier/prettier From d99fd00bc2e5c4a0afeb2d6717dac7fe77f64d33 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 12 Feb 2020 00:14:11 -0500 Subject: [PATCH 04/20] Flip cache order to mirror how it is done downstream. --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4953f7c..76801a8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,12 +14,6 @@ jobs: - uses: actions/setup-python@v1 with: python-version: 3.8 - - name: Cache pre-commit hooks - uses: actions/cache@v1 - with: - path: ~/.cache/pre-commit - key: "${{ runner.os }}-pre-commit-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Cache pip test requirements uses: actions/cache@v1 with: @@ -29,6 +23,12 @@ jobs: restore-keys: | ${{ runner.os }}-pip-test- ${{ runner.os }}-pip- + - name: Cache pre-commit hooks + uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: "${{ runner.os }}-pre-commit-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" - name: Install dependencies run: | python -m pip install --upgrade pip From e96577bce4b3b6aefa044943e478301a7d11288f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 18 Feb 2020 18:04:06 -0500 Subject: [PATCH 05/20] All references to '-r' for pip calls have been replaced with the more verbose '--requirement'. --- .github/workflows/build.yml | 2 +- CONTRIBUTING.md | 2 +- requirements-dev.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 76801a8..aff7e7a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,6 +32,6 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade -r requirements-test.txt + pip install --upgrade --requirement requirements-test.txt - name: Run pre-commit on all files run: pre-commit run --all-files diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eb00ca9..dacaaad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,7 @@ commands: cd skeleton-generic pyenv virtualenv skeleton-generic pyenv local skeleton-generic -pip install -r requirements-dev.txt +pip install --requirement requirements-dev.txt ``` #### Installing the pre-commit hook #### diff --git a/requirements-dev.txt b/requirements-dev.txt index f122cc5..d84ee68 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,2 @@ --r requirements-test.txt +--requirement requirements-test.txt ipython From 067ee0850c154845b7de623988c5a1bd5ce67d3a Mon Sep 17 00:00:00 2001 From: Felddy Date: Thu, 20 Feb 2020 17:29:09 -0500 Subject: [PATCH 06/20] Autoupdate pre-commit hooks. Add mypy. --- .pre-commit-config.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7856658..46cea9e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,7 +47,7 @@ repos: additional_dependencies: - flake8-docstrings - repo: https://github.com/asottile/pyupgrade - rev: v1.26.2 + rev: v2.0.0 hooks: - id: pyupgrade - repo: https://github.com/PyCQA/bandit @@ -74,7 +74,7 @@ repos: rev: v4.2.0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml + # files: molecule/default/playbook.yml - repo: https://github.com/antonbabenko/pre-commit-terraform.git rev: v1.12.0 hooks: @@ -88,3 +88,7 @@ repos: rev: 1.19.1 hooks: - id: prettier + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.761 + hooks: + - id: mypy From cb7b74ea67eb0b591a1e0ade1e520b55470ce916 Mon Sep 17 00:00:00 2001 From: Felddy Date: Fri, 21 Feb 2020 17:35:07 -0500 Subject: [PATCH 07/20] Add script to create GitHub Actions secrets. --- project_setup/scripts/iam-to-github | 149 ++++++++++++++++++++++++++++ setup.py | 3 +- 2 files changed, 151 insertions(+), 1 deletion(-) create mode 100755 project_setup/scripts/iam-to-github diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github new file mode 100755 index 0000000..3c94b8b --- /dev/null +++ b/project_setup/scripts/iam-to-github @@ -0,0 +1,149 @@ +#!/usr/bin/env python + +"""Extract AWS credentials from terraform state, encrypt, and upload to GitHub. + +This command must be executed in the directory containing the .terraform state +within the a GitHub project. + +Usage: + iam-to-github [--log-level=LEVEL] + iam-to-github (-h | --help) + +Options: + -h --help Show this message. + --log-level=LEVEL If specified, then the log level will be set to + the specified value. Valid values are "debug", "info", + "warning", "error", and "critical". [default: info] +""" + +# Standard Python Libraries +from base64 import b64encode +import json +import logging +import subprocess # nosec +import sys +from typing import Dict + +# Third-Party Libraries +import docopt + +# cisagov Libraries +from nacl import encoding, public +import requests + + +def creds_from_child(child_module): + """Search for IAM access keys in child resources. + + Returns (key_id, secret) if found, (None, None) otherwise. + """ + for resource in child_module["resources"]: + if resource["address"] == "aws_iam_access_key.key": + key_id = resource["values"]["id"] + secret = resource["values"]["secret"] + return key_id, secret + return None, None + + +def creds_from_terraform(): + """Retrieve IAM credentials from terraform state. + + Returns (key_id, secret) if found, (None, None) otherwise. + """ + c = subprocess.run( # nosec + "terraform show --json", shell=True, stdout=subprocess.PIPE # nosec + ) + j = json.loads(c.stdout) + + if not j.get("values"): + return None, None + + for child_module in j["values"]["root_module"]["child_modules"]: + key_id, secret = creds_from_child(child_module) + if key_id: + return key_id, secret + else: + return None, None + + +def encrypt(public_key: str, secret_value: str) -> str: + """Encrypt a Unicode string using the public key.""" + public_key = public.PublicKey(public_key.encode("utf-8"), encoding.Base64Encoder()) + sealed_box = public.SealedBox(public_key) + encrypted = sealed_box.encrypt(secret_value.encode("utf-8")) + return b64encode(encrypted).decode("utf-8") + + +def get_public_key(session: requests.Session, repo_name) -> Dict[str, str]: + """Fetch the public key for a repository.""" + logging.info(f"Requesting public key for repository {repo_name}") + response = session.get( + f"https://api.github.com/repos/{repo_name}/actions/secrets/public-key" + ) + response.raise_for_status() + return response.json() + + +def set_secret( + session: requests.Session, + repo_name: str, + secret_name: str, + secret_value: str, + public_key: Dict[str, str], +) -> None: + """Create a secret in the repository.""" + logging.info(f"Creating secret {secret_name}") + encrypted_secret_value = encrypt(public_key["key"], secret_value) + response = session.put( + f"https://api.github.com/repos/{repo_name}/actions/secrets/{secret_name}", + json={ + "encrypted_value": encrypted_secret_value, + "key_id": public_key["key_id"], + }, + ) + response.raise_for_status() + + +def main(): + """Set up logging and call the requested commands.""" + args = docopt.docopt(__doc__, version="0.0.1") + + # Set up logging + log_level = args["--log-level"] + try: + logging.basicConfig( + format="%(asctime)-15s %(levelname)s %(message)s", level=log_level.upper() + ) + except ValueError: + logging.critical( + f'"{log_level}" is not a valid logging level. Possible values ' + "are debug, info, warning, and error." + ) + return 1 + + github_token: str = args[""] + repo_name: str = args[""] + + logging.info("Searching Terraform state for IAM credentials.") + aws_key_id: str + aws_secret: str + aws_key_id, aws_secret = creds_from_terraform() + if aws_key_id is None: + logging.error("Credentials not found in terraform state.") + logging.error("Is there a .terraform state directory here?") + sys.exit(-1) + + logging.info("Creating GitHub API session.") + session: requests.Session = requests.Session() + session.auth = (None, github_token) + + public_key: Dict[str, str] = get_public_key(session, repo_name) + + set_secret(session, repo_name, "AWS_ACCESS_KEY_ID", aws_key_id, public_key) + set_secret(session, repo_name, "AWS_SECRET_ACCESS_KEY", aws_secret, public_key) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/setup.py b/setup.py index b5ea6e3..d33eb16 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def package_vars(version_file): python_requires=">=3.6", # What does your project relate to? keywords="documentation", - install_requires=["docopt", "setuptools >= 24.2.0", "schema", "PyGithub"], + install_requires=["docopt", "PyNaCl", "setuptools >= 24.2.0", "schema", "PyGithub"], extras_require={ "test": [ "pre-commit", @@ -78,6 +78,7 @@ def package_vars(version_file): scripts=[ "project_setup/scripts/ansible-roles", "project_setup/scripts/iam-to-travis", + "project_setup/scripts/iam-to-github", "project_setup/scripts/skeleton", "project_setup/scripts/ssm-param", ], From c9c573154d47eb5c26fe77963ba59735c94d93b3 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sat, 22 Feb 2020 13:19:00 -0500 Subject: [PATCH 08/20] Correct isort config after seed-hook borked it. --- .isort.cfg | 2 +- project_setup/scripts/iam-to-github | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index a4b747a..ea88304 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,6 +7,6 @@ import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries # Should be auto-populated by seed-isort-config hook -known_third_party=setuptools +known_third_party=docopt,github,nacl,requests,setuptools # These must be manually set to correctly separate them from third party libraries known_first_party= diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 45601b2..4523932 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -24,7 +24,7 @@ import subprocess # nosec import sys from typing import Dict -# cisagov Libraries +# Third-Party Libraries import docopt from nacl import encoding, public import requests From 871a1a2b7bd724380324a498df755151c6731257 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sat, 22 Feb 2020 15:44:27 -0500 Subject: [PATCH 09/20] Add detection of GitHub repo name and extra users. Adds the ability to filter users. Adds better pre-launch checks. Adds ability to add suffixes to secret names. --- .isort.cfg | 2 +- project_setup/scripts/iam-to-github | 189 +++++++++++++++++++++------- 2 files changed, 145 insertions(+), 46 deletions(-) diff --git a/.isort.cfg b/.isort.cfg index ea88304..4321048 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -7,6 +7,6 @@ import_heading_thirdparty=Third-Party Libraries import_heading_firstparty=cisagov Libraries # Should be auto-populated by seed-isort-config hook -known_third_party=docopt,github,nacl,requests,setuptools +known_third_party=docopt,github,nacl,requests,setuptools,schema # These must be manually set to correctly separate them from third party libraries known_first_party= diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 4523932..3420d25 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -3,10 +3,13 @@ """Extract AWS credentials from terraform state, encrypt, and upload to GitHub. This command must be executed in the directory containing the .terraform state -within the a GitHub project. +within a GitHub project. It will attempt to detect the repository name from +the projects origin. + +It requires a Personal Access Token from GitHub that has "repo" access scope. Usage: - iam-to-github [--log-level=LEVEL] + iam-to-github [options] iam-to-github (-h | --help) Options: @@ -14,54 +17,63 @@ Options: --log-level=LEVEL If specified, then the log level will be set to the specified value. Valid values are "debug", "info", "warning", "error", and "critical". [default: info] + --repo=REPONAME Use provided repository name instead of detecting it. + --suffix=SUFFIX Append a suffix to the secret names. + --user=FILTER Only process users that contain the filter text. """ # Standard Python Libraries from base64 import b64encode import json import logging -import subprocess # nosec +import re +import subprocess # nosec : security implications have been considered import sys -from typing import Dict +from typing import Any, Dict, Generator, List, Optional, Tuple # Third-Party Libraries import docopt from nacl import encoding, public import requests +from schema import And, Or, Schema, SchemaError, Use + +GIT_URL_RE: re.Pattern = re.compile("git@github.com:(.*).git") -def creds_from_child(child_module): +def creds_from_child(child_module: Dict) -> Generator[Tuple[str, str, str], None, None]: """Search for IAM access keys in child resources. - Returns (key_id, secret) if found, (None, None) otherwise. + Yields (user, key_id, secret) when found. """ for resource in child_module["resources"]: if resource["address"] == "aws_iam_access_key.key": - key_id = resource["values"]["id"] - secret = resource["values"]["secret"] - return key_id, secret - return None, None + key_id: str = resource["values"]["id"] + secret: str = resource["values"]["secret"] + user: str = resource["values"]["user"] + yield user, key_id, secret + return -def creds_from_terraform(): +def creds_from_terraform() -> Generator[Tuple[str, str, str], None, None]: """Retrieve IAM credentials from terraform state. - Returns (key_id, secret) if found, (None, None) otherwise. + Yields (user, key_id, secret) when found. """ - c = subprocess.run( # nosec - "terraform show --json", shell=True, stdout=subprocess.PIPE # nosec - ) - j = json.loads(c.stdout) + user: str + key_id: str + secret: str + c = subprocess.run(["terraform", "show", "--json"], stdout=subprocess.PIPE) # nosec + # Normally we'd check the process return code here. But terraform is perfectly + # happy to return zero even if there was no state files. + j: Dict = json.loads(c.stdout) if not j.get("values"): - return None, None + return for child_module in j["values"]["root_module"]["child_modules"]: - key_id, secret = creds_from_child(child_module) - if key_id: - return key_id, secret - else: - return None, None + for user, key_id, secret in creds_from_child(child_module): + yield user, key_id, secret + return def encrypt(public_key: str, secret_value: str) -> str: @@ -89,7 +101,7 @@ def set_secret( secret_value: str, public_key: Dict[str, str], ) -> None: - """Create a secret in the repository.""" + """Create a secret in a repository.""" logging.info(f"Creating secret {secret_name}") encrypted_secret_value = encrypt(public_key["key"], secret_value) response = session.put( @@ -102,43 +114,130 @@ def set_secret( response.raise_for_status() +def get_repo_name() -> str: + """Get the repository name using git.""" + logging.debug(f"Trying to determine GitHub repository name using git.") + c = subprocess.run( # nosec + ["git", "remote", "get-url", "origin"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if c.returncode != 0: + logging.critical("Could not determine GitHub repository name.") + raise Exception(c.stderr) + match = GIT_URL_RE.match(c.stdout.decode()) + repo_name = match.groups()[0] # type: ignore + return repo_name + + def main() -> int: """Set up logging and call the requested commands.""" - args = docopt.docopt(__doc__, version="0.0.1") + args: Dict[str, str] = docopt.docopt(__doc__, version="1.0.0") + + # Validate and convert arguments as needed + schema: Schema = Schema( + { + "": And( + str, + lambda n: len(n) == 40, + error=" must be a 40 character personal access token.", + ), + "--log-level": And( + str, + Use(str.lower), + lambda n: n in ("debug", "info", "warning", "error", "critical"), + error="Possible values for --log-level are " + + "debug, info, warning, error, and critical.", + ), + "--repo": Or( + None, + And( + str, + lambda n: "/" in n, + error='Repository names must contain a "/"', + ), + ), + "--suffix": Or(And(None, Use(lambda n: "")), And(str, Use(str.upper))), + str: object, # Don't care about other keys, if any + } + ) - # Set up logging - log_level = args["--log-level"] try: - logging.basicConfig( - format="%(asctime)-15s %(levelname)s %(message)s", level=log_level.upper() - ) - except ValueError: - logging.critical( - f'"{log_level}" is not a valid logging level. Possible values ' - "are debug, info, warning, and error." - ) + validated_args: Dict[str, Any] = schema.validate(args) + except SchemaError as err: + # Exit because one or more of the arguments were invalid + print(err, file=sys.stderr) return 1 - github_token: str = args[""] - repo_name: str = args[""] + # Assign validated arguments to variables + github_token: str = validated_args[""] + log_level: str = validated_args["--log-level"] + repo_name: str = validated_args["--repo"] + secret_suffix: str = validated_args["--suffix"] + user_filter: str = validated_args["--user"] + + # Set up logging + logging.basicConfig( + format="%(asctime)-15s %(levelname)s %(message)s", level=log_level.upper() + ) + + # If the user does not provide a repo name we'll try to determine it from git + if not repo_name: + repo_name = get_repo_name() + logging.info(f"Using GitHub repository name: {repo_name}") + + if user_filter: + logging.info(f"Filter to users containing: {user_filter}") + + if secret_suffix: + if not secret_suffix.startswith("_"): + secret_suffix = f"_{secret_suffix}" + logging.info(f'Apending "{secret_suffix}" to secret names.') + + aws_user: Optional[str] = None + aws_key_id: Optional[str] = None + aws_secret: Optional[str] = None + cred_list: List[Tuple[str, str, str]] = [] logging.info("Searching Terraform state for IAM credentials.") - aws_key_id: str - aws_secret: str - aws_key_id, aws_secret = creds_from_terraform() - if aws_key_id is None: - logging.error("Credentials not found in terraform state.") - logging.error("Is there a .terraform state directory here?") + for aws_user, aws_key_id, aws_secret in creds_from_terraform(): + logging.info(f"Found credentials for user: {aws_user}") + if not user_filter or user_filter in aws_user: + logging.debug(f"User {aws_user} matches filter {user_filter}") + cred_list.append((aws_user, aws_key_id, aws_secret)) + + if len(cred_list) == 0: + logging.error("No credentials matched in terraform state.") + if aws_user: # We found a user but it wasn't used + logging.error( + "Users found in Terraform state but were filtered out by --user" + ) + else: # We never saw a user + logging.error("Is there a .terraform state directory here?") sys.exit(-1) - logging.info("Creating GitHub API session.") + if len(cred_list) > 1: + logging.error("Too many credentials found. Use --user to match one.") + sys.exit(-2) + + # All the ducks are in a row, let's do this thang! + logging.info("Creating GitHub API session using personal access token.") session: requests.Session = requests.Session() session.auth = ("", github_token) public_key: Dict[str, str] = get_public_key(session, repo_name) - set_secret(session, repo_name, "AWS_ACCESS_KEY_ID", aws_key_id, public_key) - set_secret(session, repo_name, "AWS_SECRET_ACCESS_KEY", aws_secret, public_key) + logging.info(f"Setting secrets for user: {aws_user}") + set_secret( + session, repo_name, "AWS_ACCESS_KEY_ID" + secret_suffix, aws_key_id, public_key + ) + set_secret( + session, + repo_name, + "AWS_SECRET_ACCESS_KEY" + secret_suffix, + aws_secret, + public_key, + ) return 0 From ed1c4cc171a44940bd7a2b650cb93565abb745ab Mon Sep 17 00:00:00 2001 From: Felddy Date: Sat, 22 Feb 2020 15:51:30 -0500 Subject: [PATCH 10/20] Clarify tool documentation. --- project_setup/scripts/iam-to-github | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 3420d25..113592b 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -1,10 +1,10 @@ #!/usr/bin/env python -"""Extract AWS credentials from terraform state, encrypt, and upload to GitHub. +"""Create GitHub secrets from AWS credentials extracted from Terraform state. This command must be executed in the directory containing the .terraform state within a GitHub project. It will attempt to detect the repository name from -the projects origin. +the projects git origin. It requires a Personal Access Token from GitHub that has "repo" access scope. From 4de5ee68ac4852d18cc870320e6dfd111b2965be Mon Sep 17 00:00:00 2001 From: Felddy Date: Sat, 22 Feb 2020 16:30:51 -0500 Subject: [PATCH 11/20] Replace travis docs with GitHub actions docs. --- project_setup/README.md | 56 +++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/project_setup/README.md b/project_setup/README.md index 822dc68..208a5f2 100644 --- a/project_setup/README.md +++ b/project_setup/README.md @@ -110,50 +110,34 @@ usage of the tool is: This file will now contain definitions for all the Ansible roles. Edit the file, and remove any role that will not be required for your project. -## Terraform IAM Credentials to Travis Tool 🔑‍👉👷🏻 ## +## Terraform IAM Credentials to GitHub Secrets 🔑‍👉🤫 ## -When Travis-CI needs credentials to run we provide them in its `.travis.yml` -file in an encrypted format. Extracting fresh IAM credentials from a -Terraform run, -[encrypting them properly](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml) -, and then formatting them into well-formed `yml` that will make the linters -happy is no small task. +When GitHub Actions workflows require credentials to run we provide them via +secrets. This usually involves extracting the secrets from the Terraform state +json output. Then some pointing, clicking, cutting and pasting on the +repository's settings. -To simplify this task use the [`iam-to-travis`](scripts/iam-to-travis) tool -located in the [`scripts`](scripts) directory. The tool will output `yml` -that can be pasted directly into a `.travis.yml` file. The tool has -adjustable indentation and width with reasonable defaults. +To simplify this task use the [`iam-to-github`](scripts/iam-to-github) tool +located in the [`scripts`](scripts) directory. The tool will create secrets +using your +[personal access token (PAT)](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). +Note: Your PAT requires the "repo" scope to be set. Execute the tool from your GitHub project's terraform directory: ```bash -iam-to-travis +iam-to-github xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx ``` -```yml -# AWS_ACCESS_KEY_ID -- secure: "UZVXTpslA2qID+51nIlVOPaDZNdIuTuEw7AMG7045zEAuxtmViHBKL5z6fpwUnmV\ - vqnjZksQGLuLnYFIX+E85ObwrFBjhnr0m5baARG3wdS3+KrQjruStetz8gpKeXLA+L81VfhxF\ - Z3pCTATN86FSEcVFSgA8hGGr1nKqpVKFqxnKCXnBeQCgqD0MrQ1bfj7AvhY1s97pf6kXBwQ7/\ - MoRhFgwcnituhhDpb6QZWFo6L6W/UUKL5sATA3H2tGSLtS8W8a0MMTKr6n8uSoiQwNO8+qGvm\ - Iu10tl+XPzHzYGdpmBZignA63HFOcodtDy3PJJzICrEmkGZ9zvp0kgtHi7hBLXSLf82D28lMt\ - r8QC+ZCnuaH+ar0SaSueDS7MkUVXl0DNIkqjQZP4/AwTuAomJ4i60cpPS3Xte8GO8dUqwkc+/\ - 2mg1cqeUQAVaIMsFzs3U11LfC5CMzHNe9qe8jRt6aVylxpdXEpxLeD3kNG3mBEGxOAQD0YkYw\ - tp1paKJOt2CHEnTkDrs8hJ9bZzlwGrmu0vIfFoO0k1/rYbemNGZ+VCY1TWtxOdqxeJRLVYZBJ\ - J0cTlYf5N3//RUZS6QZ85dJ/7zKn4SRjalLMAD/zjAte5EsRag34KWG6LurRojKqpPUfzaxRu\ - IEYQ1+ATLZoEgMLYETiEWG6Il2QNv1c6uOM=" - -# AWS_SECRET_ACCESS_KEY -- secure: "PcuGEOpW7f448RSA6TB07EwI2IlcCoWkrjztO8zz34rKE2VYNTaNEseZizTg0B0p\ - s80jvlJRfkuQi+h0nYsLCANjsX+o1HooXnNBFsREzuKhYu1qB80Tcpl3DY/uYXF0yLbe0Qk0s\ - ZmDxK3Fe62bjLzlMTh2i5Aocf+e176zQ+VrJqHG4qSVrgRPXeRcKRrKFyOYA/HbmC7Wcno85d\ - nsj0s3U4sDxrn6rWaHetHFxEml5kD86XhJ4xKXhZfwCR+aVgvKEdiY4ft6wmfbogVhAqfa5NG\ - N4CrCs1ooKutB/95axlmuxEG73mnYdBaE2FphOvx+2lL8JOVjtUK5ENac1QumHngztAASTtVc\ - RXpEaRH5OGEgWkmqptN3fJAqZyfLu74zOr61/thJuh6fkAciXDoKt8e2CyCxAqbAB+6SKwxG3\ - +K6rEtms6c3dwtHrssoHOsozADxVeK/I2two1QzcVsw92hRfF9ecWyV+QUaJ6iZYEk2VqgsDi\ - NuBbVa2SQT9mO3A4fcn23fRjHy/ac/Cmz9q3hGKnMWl27CSRaPq7PR4sNPr9ebabRRrjAZ0I2\ - UaWaDqIOwz85EWTQ6Y/53dgr2Zgv8KpfzfdNWhKtKS4woJGYPoU1k17V2TGZhs2S85XfT2aB3\ - injrwJ5qqmcUljFdByuA08WyX4UkBCWtkJE=" +```console +2020-02-22 15:50:36,059 INFO Using GitHub repository name: cisagov/ansible-role-dev-ssh-access +2020-02-22 15:50:36,060 INFO Searching Terraform state for IAM credentials. +2020-02-22 15:50:40,643 INFO Found credentials for user: test-ansible-role-dev-ssh-access +2020-02-22 15:50:40,643 INFO Creating GitHub API session using personal access token. +2020-02-22 15:50:40,644 INFO Requesting public key for repository cisagov/ansible-role-dev-ssh-access +2020-02-22 15:50:40,832 INFO Setting secrets for user: test-ansible-role-dev-ssh-access +2020-02-22 15:50:40,832 INFO Creating secret AWS_ACCESS_KEY_ID +2020-02-22 15:50:41,027 INFO Creating secret AWS_SECRET_ACCESS_KEY ``` ## Managing SSM Parameters from Files 🗂👉☁️ ## From c2e9711b974e240edb8847800e82d6bd9327c2b7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sat, 22 Feb 2020 16:31:37 -0500 Subject: [PATCH 12/20] Qapla'! --- project_setup/scripts/iam-to-github | 1 + 1 file changed, 1 insertion(+) diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 113592b..51b5645 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -238,6 +238,7 @@ def main() -> int: aws_secret, public_key, ) + logging.info("Success!") return 0 From bc8dfe5ddfff2a6ec75d8cc7458499d3378e609d Mon Sep 17 00:00:00 2001 From: Felddy Date: Sat, 22 Feb 2020 16:33:09 -0500 Subject: [PATCH 13/20] Add link to secrets docs. --- project_setup/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project_setup/README.md b/project_setup/README.md index 208a5f2..7c490e3 100644 --- a/project_setup/README.md +++ b/project_setup/README.md @@ -113,7 +113,8 @@ the file, and remove any role that will not be required for your project. ## Terraform IAM Credentials to GitHub Secrets 🔑‍👉🤫 ## When GitHub Actions workflows require credentials to run we provide them via -secrets. This usually involves extracting the secrets from the Terraform state +[secrets](https://help.github.com/en/actions/configuring-and-managing-workflows/creating-and-storing-encrypted-secrets). +This usually involves extracting the secrets from the Terraform state json output. Then some pointing, clicking, cutting and pasting on the repository's settings. From bac7c9d7ec2b5ce8fff251818ecba1e4b051e802 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 23 Feb 2020 09:58:14 -0500 Subject: [PATCH 14/20] Clarify invocation of tool with bogus PAT. --- project_setup/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project_setup/README.md b/project_setup/README.md index 7c490e3..ec81184 100644 --- a/project_setup/README.md +++ b/project_setup/README.md @@ -122,12 +122,12 @@ To simplify this task use the [`iam-to-github`](scripts/iam-to-github) tool located in the [`scripts`](scripts) directory. The tool will create secrets using your [personal access token (PAT)](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line). -Note: Your PAT requires the "repo" scope to be set. +Note: Your PAT needs to have the "repo" scope set. Execute the tool from your GitHub project's terraform directory: ```bash -iam-to-github xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +iam-to-github 9f4ae878de917c7cf191b9861d3c1cf9224939f7 ``` ```console From 18cda9463081e3019013f895a529d5560f8f6517 Mon Sep 17 00:00:00 2001 From: Felddy Date: Sun, 23 Feb 2020 10:02:59 -0500 Subject: [PATCH 15/20] Qapla'! --- project_setup/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/project_setup/README.md b/project_setup/README.md index ec81184..d1a5c9c 100644 --- a/project_setup/README.md +++ b/project_setup/README.md @@ -139,6 +139,7 @@ iam-to-github 9f4ae878de917c7cf191b9861d3c1cf9224939f7 2020-02-22 15:50:40,832 INFO Setting secrets for user: test-ansible-role-dev-ssh-access 2020-02-22 15:50:40,832 INFO Creating secret AWS_ACCESS_KEY_ID 2020-02-22 15:50:41,027 INFO Creating secret AWS_SECRET_ACCESS_KEY +2020-02-22 15:50:41,036 INFO Success! ``` ## Managing SSM Parameters from Files 🗂👉☁️ ## From ff5560884f162b31acdaaa4614fe618b35ea7692 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 24 Feb 2020 10:04:37 -0500 Subject: [PATCH 16/20] Add support for https GitHub clones. --- project_setup/scripts/iam-to-github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 51b5645..57af5d7 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -37,7 +37,7 @@ from nacl import encoding, public import requests from schema import And, Or, Schema, SchemaError, Use -GIT_URL_RE: re.Pattern = re.compile("git@github.com:(.*).git") +GIT_URL_RE: re.Pattern = re.compile("(?:git@|https://)github.com[:/](.*).git") def creds_from_child(child_module: Dict) -> Generator[Tuple[str, str, str], None, None]: From 381953d7e16c3f2e6e121f0485ff8ac4bc79be63 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 24 Feb 2020 10:06:15 -0500 Subject: [PATCH 17/20] Normalize capitalization of "Terraform" app and ".terraform" dir --- project_setup/scripts/iam-to-github | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 57af5d7..84cf17d 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -55,7 +55,7 @@ def creds_from_child(child_module: Dict) -> Generator[Tuple[str, str, str], None def creds_from_terraform() -> Generator[Tuple[str, str, str], None, None]: - """Retrieve IAM credentials from terraform state. + """Retrieve IAM credentials from Terraform state. Yields (user, key_id, secret) when found. """ @@ -63,8 +63,8 @@ def creds_from_terraform() -> Generator[Tuple[str, str, str], None, None]: key_id: str secret: str c = subprocess.run(["terraform", "show", "--json"], stdout=subprocess.PIPE) # nosec - # Normally we'd check the process return code here. But terraform is perfectly - # happy to return zero even if there was no state files. + # Normally we'd check the process return code here. But Terraform is perfectly + # happy to return zero even if there were no state files. j: Dict = json.loads(c.stdout) if not j.get("values"): @@ -207,7 +207,7 @@ def main() -> int: cred_list.append((aws_user, aws_key_id, aws_secret)) if len(cred_list) == 0: - logging.error("No credentials matched in terraform state.") + logging.error("No credentials matched in Terraform state.") if aws_user: # We found a user but it wasn't used logging.error( "Users found in Terraform state but were filtered out by --user" From 3a6a0312f35e4e0e0aa43719741e161900c1cb79 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 24 Feb 2020 10:07:31 -0500 Subject: [PATCH 18/20] Fix grammar and make reference to .terraform being a directory. --- project_setup/scripts/iam-to-github | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 84cf17d..812b8f5 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -3,8 +3,8 @@ """Create GitHub secrets from AWS credentials extracted from Terraform state. This command must be executed in the directory containing the .terraform state -within a GitHub project. It will attempt to detect the repository name from -the projects git origin. +directory within a GitHub project. It will attempt to detect the repository +name from the project's git origin. It requires a Personal Access Token from GitHub that has "repo" access scope. From c25582295a3f1ba6c910570a363824db93bb2eb7 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 24 Feb 2020 10:08:15 -0500 Subject: [PATCH 19/20] Fix misspelled word. --- project_setup/scripts/iam-to-github | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index 812b8f5..ff3b7df 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -192,7 +192,7 @@ def main() -> int: if secret_suffix: if not secret_suffix.startswith("_"): secret_suffix = f"_{secret_suffix}" - logging.info(f'Apending "{secret_suffix}" to secret names.') + logging.info(f'Appending "{secret_suffix}" to secret names.') aws_user: Optional[str] = None aws_key_id: Optional[str] = None From 2acefd086b8707e33aadbfcc73143252a97dbd55 Mon Sep 17 00:00:00 2001 From: Felddy Date: Mon, 24 Feb 2020 10:09:01 -0500 Subject: [PATCH 20/20] Raise exception if detection of repo name fails. --- project_setup/scripts/iam-to-github | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/project_setup/scripts/iam-to-github b/project_setup/scripts/iam-to-github index ff3b7df..59afd96 100755 --- a/project_setup/scripts/iam-to-github +++ b/project_setup/scripts/iam-to-github @@ -126,7 +126,12 @@ def get_repo_name() -> str: logging.critical("Could not determine GitHub repository name.") raise Exception(c.stderr) match = GIT_URL_RE.match(c.stdout.decode()) - repo_name = match.groups()[0] # type: ignore + if match: + repo_name = match.groups()[0] # type: ignore + else: + logging.critical("Could not determine GitHub repository name.") + logging.critical("Use the --repo option to specify it manually.") + raise Exception("Could not determine GitHub repository name.") return repo_name