From e0ab5062839ec83ece995758188c0026d1a579b6 Mon Sep 17 00:00:00 2001 From: Dave Redmin Date: Sun, 12 Apr 2020 17:28:02 -0400 Subject: [PATCH 1/4] Make a separate function for extracting a tagged secret from a resource. --- project_setup/scripts/terraform-to-secrets | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/project_setup/scripts/terraform-to-secrets b/project_setup/scripts/terraform-to-secrets index 7bf3eba..7fb01ed 100755 --- a/project_setup/scripts/terraform-to-secrets +++ b/project_setup/scripts/terraform-to-secrets @@ -98,6 +98,42 @@ def get_terraform_state(filename: str = None) -> Dict: return json_state +def find_tagged_secret(resource_name: str, resource_data: Dict) -> Tuple[str, str]: + """Extract a tagged secret from a resource.""" + # Ensure "tags" key exists in resource_data and if it does, make sure + # its value is not None. Both of these cases can occur. + tags: Dict[str, str] + if "tags" not in resource_data or resource_data.get("tags") is None: + tags = dict() + else: + tags = resource_data["tags"] + + secret_name: Optional[str] = tags.get(GITHUB_SECRET_NAME_TAG) + lookup_tag: Optional[str] = tags.get(GITHUB_SECRET_TERRAFORM_LOOKUP_TAG) + secret_value: Optional[str] + if secret_name: + logging.debug( + f"Found {GITHUB_SECRET_NAME_TAG} on {resource_name} " + f"with value {secret_name}" + ) + if lookup_tag: + logging.debug( + f"Found {GITHUB_SECRET_TERRAFORM_LOOKUP_TAG} on " + f"{resource_name} with value {lookup_tag}" + ) + secret_value = resource_data.get(lookup_tag) + if secret_value is None: + logging.warning(f"Could not lookup value with key {lookup_tag}") + else: + logging.debug(f"Looked up value: {secret_value}") + return secret_name, secret_value + else: + logging.warning( + f"Missing {GITHUB_SECRET_TERRAFORM_LOOKUP_TAG} on " f"{resource_name}." + ) + return "", "" + + def find_resources( terraform_state: Dict, resource_type: Optional[str] ) -> Generator[Dict, None, None]: From e9149840aca44dcebbdd753696de2f1f45517b9b Mon Sep 17 00:00:00 2001 From: Dave Redmin Date: Sun, 12 Apr 2020 17:28:56 -0400 Subject: [PATCH 2/4] Use new find_tagged_secret() function. --- project_setup/scripts/terraform-to-secrets | 34 +++------------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/project_setup/scripts/terraform-to-secrets b/project_setup/scripts/terraform-to-secrets index 7fb01ed..7885795 100755 --- a/project_setup/scripts/terraform-to-secrets +++ b/project_setup/scripts/terraform-to-secrets @@ -165,37 +165,11 @@ def parse_tagged_resources( ) -> Generator[Tuple[str, str], None, None]: """Search all resources for tags requesting the creation of a secret.""" for resource in find_resources(terraform_state, None): - # Ensure "tags" key exists in resource["values"] and if it does, - # make sure its value is not None. Both of these cases can occur. - tags: Dict[str, str] - if "tags" not in resource["values"] or resource["values"].get("tags") is None: - tags = dict() - else: - tags = resource["values"]["tags"] - secret_name: Optional[str] = tags.get(GITHUB_SECRET_NAME_TAG) - lookup_tag: Optional[str] = tags.get(GITHUB_SECRET_TERRAFORM_LOOKUP_TAG) - secret_value: str + secret_name, secret_value = find_tagged_secret( + resource["address"], resource["values"] + ) if secret_name: - logging.debug( - f"Found {GITHUB_SECRET_NAME_TAG} on {resource['address']} " - f"with value {secret_name}" - ) - if lookup_tag: - logging.debug( - f"Found {GITHUB_SECRET_TERRAFORM_LOOKUP_TAG} on " - f"{resource['address']} with value {lookup_tag}" - ) - secret_value = resource["values"].get(lookup_tag) - if secret_value is None: - logging.warning(f"Could not lookup value with key {lookup_tag}") - else: - logging.debug(f"Looked up value: {secret_value}") - yield secret_name, secret_value - else: - logging.warning( - f"Missing {GITHUB_SECRET_TERRAFORM_LOOKUP_TAG} on " - f"{resource['address']}." - ) + yield secret_name, secret_value return From 21e93782690aaac4c3fff34fa4deb4e591fba4be Mon Sep 17 00:00:00 2001 From: Dave Redmin Date: Sun, 12 Apr 2020 17:30:05 -0400 Subject: [PATCH 3/4] Search for tagged secrets in resources with outputs. --- project_setup/scripts/terraform-to-secrets | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/project_setup/scripts/terraform-to-secrets b/project_setup/scripts/terraform-to-secrets index 7885795..fa92a9c 100755 --- a/project_setup/scripts/terraform-to-secrets +++ b/project_setup/scripts/terraform-to-secrets @@ -134,6 +134,25 @@ def find_tagged_secret(resource_name: str, resource_data: Dict) -> Tuple[str, st return "", "" +def find_outputs(terraform_state: Dict) -> Generator[Dict, None, None]: + """Search for resources with outputs in the Terraform state.""" + for resource in terraform_state["values"]["root_module"]["resources"]: + if resource.get("values", dict()).get("outputs", dict()): + yield resource["values"]["outputs"] + + +def parse_tagged_outputs( + terraform_state: Dict, +) -> Generator[Tuple[str, str], None, None]: + """Search all outputs for tags requesting the creation of a secret.""" + for outputs in find_outputs(terraform_state): + for output_name, output_data in outputs.items(): + secret_name, secret_value = find_tagged_secret(output_name, output_data) + if secret_name: + yield secret_name, secret_value + return + + def find_resources( terraform_state: Dict, resource_type: Optional[str] ) -> Generator[Dict, None, None]: @@ -259,6 +278,9 @@ def get_resource_secrets(terraform_state: Dict) -> Dict[str, str]: for secret_name, secret_value in parse_tagged_resources(terraform_state): logging.info(f"Found secret: {secret_name}") secrets[secret_name] = secret_value + for secret_name, secret_value in parse_tagged_outputs(terraform_state): + logging.info(f"Found secret: {secret_name}") + secrets[secret_name] = secret_value return secrets From 7028f336701c68781a35b6c97ea5421f4a5965ec Mon Sep 17 00:00:00 2001 From: Dave Redmin Date: Mon, 13 Apr 2020 09:53:46 -0400 Subject: [PATCH 4/4] Return a generator from find_tagged_secret() to match design pattern used throughout the rest of this script. --- project_setup/scripts/terraform-to-secrets | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/project_setup/scripts/terraform-to-secrets b/project_setup/scripts/terraform-to-secrets index fa92a9c..e97297c 100755 --- a/project_setup/scripts/terraform-to-secrets +++ b/project_setup/scripts/terraform-to-secrets @@ -98,7 +98,9 @@ def get_terraform_state(filename: str = None) -> Dict: return json_state -def find_tagged_secret(resource_name: str, resource_data: Dict) -> Tuple[str, str]: +def find_tagged_secret( + resource_name: str, resource_data: Dict +) -> Generator[Tuple[str, str], None, None]: """Extract a tagged secret from a resource.""" # Ensure "tags" key exists in resource_data and if it does, make sure # its value is not None. Both of these cases can occur. @@ -126,12 +128,12 @@ def find_tagged_secret(resource_name: str, resource_data: Dict) -> Tuple[str, st logging.warning(f"Could not lookup value with key {lookup_tag}") else: logging.debug(f"Looked up value: {secret_value}") - return secret_name, secret_value + yield secret_name, secret_value else: logging.warning( f"Missing {GITHUB_SECRET_TERRAFORM_LOOKUP_TAG} on " f"{resource_name}." ) - return "", "" + return def find_outputs(terraform_state: Dict) -> Generator[Dict, None, None]: @@ -147,8 +149,9 @@ def parse_tagged_outputs( """Search all outputs for tags requesting the creation of a secret.""" for outputs in find_outputs(terraform_state): for output_name, output_data in outputs.items(): - secret_name, secret_value = find_tagged_secret(output_name, output_data) - if secret_name: + for secret_name, secret_value in find_tagged_secret( + output_name, output_data + ): yield secret_name, secret_value return @@ -184,10 +187,9 @@ def parse_tagged_resources( ) -> Generator[Tuple[str, str], None, None]: """Search all resources for tags requesting the creation of a secret.""" for resource in find_resources(terraform_state, None): - secret_name, secret_value = find_tagged_secret( + for secret_name, secret_value in find_tagged_secret( resource["address"], resource["values"] - ) - if secret_name: + ): yield secret_name, secret_value return