From e27c946798251fdc0d10a34d48bc8f54b8a483cf Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Thu, 24 Nov 2022 14:53:33 -0800 Subject: [PATCH 1/9] Update transform test script to update ManagedPolicyArn with partion --- bin/add_transform_test.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index f37e94aa4..b7f81892a 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -4,10 +4,12 @@ Usage: add_transform_test.py --template-file=sam-template.yaml [--disable-api-configuration] add_transform_test.py --template-file=sam-template.yaml + add_transform_test.py --template-file=sam-template.yaml [--update-partition] Options: --template-file= Location of SAM template to transform [default: template.yaml]. --disable-api-configuration Disable adding REGIONAL configuration to AWS::ApiGateway::RestApi + --update-partition Update the partition of ManagedPolicyArns in IAM Role """ import json import subprocess @@ -45,6 +47,20 @@ def add_regional_endpoint_configuration_if_needed(template: Dict[str, Any]) -> D return template +def update_partition(region: str, template: Dict[str, Any]) -> Dict[str, Any]: + for _, resource in template["Resources"].items(): + if resource["Type"] == "AWS::IAM::Role": + properties = resource["Properties"] + if "ManagedPolicyArns" in properties: + ManagedPolicyArns = properties["ManagedPolicyArns"] + UpdatedArns = [] + for ManagedPolicyArn in ManagedPolicyArns: + split_arn = ManagedPolicyArn.split(":") + split_arn[1] = region + UpdatedArns.append(":".join(split_arn)) + properties["ManagedPolicyArns"] = UpdatedArns + return template + def generate_transform_test_output_files(input_file_path: str, file_basename: str) -> None: output_file_option = file_basename + ".json" @@ -67,20 +83,22 @@ def generate_transform_test_output_files(input_file_path: str, file_basename: st transform_test_output_path = os.path.join(TRANSFORM_TEST_DIR, "output", output_file_option) shutil.copyfile(temp_output_file.name, transform_test_output_path) - regional_transform_test_output_paths = [ - os.path.join(TRANSFORM_TEST_DIR, path, output_file_option) - for path in [ - "output/aws-cn/", - "output/aws-us-gov/", - ] - ] + regional_transform_test_output_paths = { + "aws-cn": os.path.join(TRANSFORM_TEST_DIR, "output/aws-cn/", output_file_option), + "aws-us-gov": os.path.join(TRANSFORM_TEST_DIR, "output/aws-us-gov/", output_file_option) + } if not CLI_OPTIONS.get("--disable-api-configuration"): template = read_json_file(temp_output_file.name) template = add_regional_endpoint_configuration_if_needed(template) write_json_file(template, temp_output_file.name) - for output_path in regional_transform_test_output_paths: + for region, output_path in regional_transform_test_output_paths.items(): + print(output_path) + if CLI_OPTIONS.get("--update-partition"): + template = read_json_file(temp_output_file.name) + template = update_partition(region, template) + write_json_file(template, temp_output_file.name) shutil.copyfile(temp_output_file.name, output_path) print(f"Transform Test output files generated {output_path}") From bd98f6598b96c2666be5dc66c73104ce0e979c5b Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Thu, 24 Nov 2022 15:10:31 -0800 Subject: [PATCH 2/9] make black --- bin/add_transform_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index b7f81892a..c69028043 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -47,6 +47,7 @@ def add_regional_endpoint_configuration_if_needed(template: Dict[str, Any]) -> D return template + def update_partition(region: str, template: Dict[str, Any]) -> Dict[str, Any]: for _, resource in template["Resources"].items(): if resource["Type"] == "AWS::IAM::Role": @@ -85,7 +86,7 @@ def generate_transform_test_output_files(input_file_path: str, file_basename: st regional_transform_test_output_paths = { "aws-cn": os.path.join(TRANSFORM_TEST_DIR, "output/aws-cn/", output_file_option), - "aws-us-gov": os.path.join(TRANSFORM_TEST_DIR, "output/aws-us-gov/", output_file_option) + "aws-us-gov": os.path.join(TRANSFORM_TEST_DIR, "output/aws-us-gov/", output_file_option), } if not CLI_OPTIONS.get("--disable-api-configuration"): From b004506dd6b04f454a4f5099bd53d5165499a38f Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Fri, 25 Nov 2022 12:34:27 -0800 Subject: [PATCH 3/9] use a simpler way to replace partition --- bin/add_transform_test.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index c69028043..888f1a307 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -4,12 +4,10 @@ Usage: add_transform_test.py --template-file=sam-template.yaml [--disable-api-configuration] add_transform_test.py --template-file=sam-template.yaml - add_transform_test.py --template-file=sam-template.yaml [--update-partition] Options: --template-file= Location of SAM template to transform [default: template.yaml]. --disable-api-configuration Disable adding REGIONAL configuration to AWS::ApiGateway::RestApi - --update-partition Update the partition of ManagedPolicyArns in IAM Role """ import json import subprocess @@ -56,9 +54,8 @@ def update_partition(region: str, template: Dict[str, Any]) -> Dict[str, Any]: ManagedPolicyArns = properties["ManagedPolicyArns"] UpdatedArns = [] for ManagedPolicyArn in ManagedPolicyArns: - split_arn = ManagedPolicyArn.split(":") - split_arn[1] = region - UpdatedArns.append(":".join(split_arn)) + replaced_arn = ManagedPolicyArn.replace("arn:aws:", f"arn:{region}:") + UpdatedArns.append(replaced_arn) properties["ManagedPolicyArns"] = UpdatedArns return template @@ -95,12 +92,10 @@ def generate_transform_test_output_files(input_file_path: str, file_basename: st write_json_file(template, temp_output_file.name) for region, output_path in regional_transform_test_output_paths.items(): - print(output_path) - if CLI_OPTIONS.get("--update-partition"): - template = read_json_file(temp_output_file.name) - template = update_partition(region, template) - write_json_file(template, temp_output_file.name) shutil.copyfile(temp_output_file.name, output_path) + template = read_json_file(output_path) + template = update_partition(region, template) + write_json_file(template, output_path) print(f"Transform Test output files generated {output_path}") From 9a30c61ff1baf97350c15c2d09b2760c899b53c4 Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Fri, 25 Nov 2022 13:34:46 -0800 Subject: [PATCH 4/9] replace the whole template --- bin/add_transform_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index 888f1a307..fef7e3eab 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -94,8 +94,9 @@ def generate_transform_test_output_files(input_file_path: str, file_basename: st for region, output_path in regional_transform_test_output_paths.items(): shutil.copyfile(temp_output_file.name, output_path) template = read_json_file(output_path) - template = update_partition(region, template) - write_json_file(template, output_path) + with open(output_path, "w") as file: + updated_template = json.loads(json.dumps(template).replace("arn:aws", f"aws:{region}")) + file.write(json.dumps(updated_template, indent=2)) print(f"Transform Test output files generated {output_path}") From dee4f62e6708ff3467408bccb7e8a562a5b902f4 Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Fri, 25 Nov 2022 13:36:02 -0800 Subject: [PATCH 5/9] Remove unused func --- bin/add_transform_test.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index fef7e3eab..5d6076875 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -46,20 +46,6 @@ def add_regional_endpoint_configuration_if_needed(template: Dict[str, Any]) -> D return template -def update_partition(region: str, template: Dict[str, Any]) -> Dict[str, Any]: - for _, resource in template["Resources"].items(): - if resource["Type"] == "AWS::IAM::Role": - properties = resource["Properties"] - if "ManagedPolicyArns" in properties: - ManagedPolicyArns = properties["ManagedPolicyArns"] - UpdatedArns = [] - for ManagedPolicyArn in ManagedPolicyArns: - replaced_arn = ManagedPolicyArn.replace("arn:aws:", f"arn:{region}:") - UpdatedArns.append(replaced_arn) - properties["ManagedPolicyArns"] = UpdatedArns - return template - - def generate_transform_test_output_files(input_file_path: str, file_basename: str) -> None: output_file_option = file_basename + ".json" From 43d46d8ce0f2275fe520ac36f182c4a76df674ab Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Fri, 25 Nov 2022 13:55:08 -0800 Subject: [PATCH 6/9] optimize --- DEVELOPMENT_GUIDE.md | 5 +++++ bin/add_transform_test.py | 18 ++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/DEVELOPMENT_GUIDE.md b/DEVELOPMENT_GUIDE.md index 91c4f3521..11677b93b 100644 --- a/DEVELOPMENT_GUIDE.md +++ b/DEVELOPMENT_GUIDE.md @@ -143,6 +143,11 @@ To disable this feature, run the following command instead. python3 bin/add_transform_test.py --template-file template.yaml --disable-api-configuration ``` +The script will update the partition of arn based on the output path. To disable updating the partition of arn +```bash +python3 bin/add_transform_test.py --template-file template.yaml --disable-update-partition +``` + Note that please always check the generated output is as expected. This tool does not guarantee correct output. diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index 5d6076875..c6e45c571 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -4,10 +4,12 @@ Usage: add_transform_test.py --template-file=sam-template.yaml [--disable-api-configuration] add_transform_test.py --template-file=sam-template.yaml + add_transform_test.py --template-file=sam-template.yaml [--disable-update-partition] Options: --template-file= Location of SAM template to transform [default: template.yaml]. --disable-api-configuration Disable adding REGIONAL configuration to AWS::ApiGateway::RestApi + --disable-update-partition Disable updating the partition of arn to aws-cn/aws-us-gov """ import json import subprocess @@ -45,6 +47,13 @@ def add_regional_endpoint_configuration_if_needed(template: Dict[str, Any]) -> D return template +def replace_aws_partition(partition: str, file_path: str): + template = read_json_file(file_path) + with open(file_path, "w") as file: + updated_template = json.loads(json.dumps(template).replace("arn:aws", f"aws:{partition}")) + file.write(json.dumps(updated_template, indent=2)) + print(f"Transform Test output files generated {file_path}") + def generate_transform_test_output_files(input_file_path: str, file_basename: str) -> None: output_file_option = file_basename + ".json" @@ -77,13 +86,10 @@ def generate_transform_test_output_files(input_file_path: str, file_basename: st template = add_regional_endpoint_configuration_if_needed(template) write_json_file(template, temp_output_file.name) - for region, output_path in regional_transform_test_output_paths.items(): + for partition, output_path in regional_transform_test_output_paths.items(): shutil.copyfile(temp_output_file.name, output_path) - template = read_json_file(output_path) - with open(output_path, "w") as file: - updated_template = json.loads(json.dumps(template).replace("arn:aws", f"aws:{region}")) - file.write(json.dumps(updated_template, indent=2)) - print(f"Transform Test output files generated {output_path}") + if not CLI_OPTIONS.get("--disable-update-partition"): + replace_aws_partition(partition, output_path) def get_input_file_path() -> str: From 52e6ab14fddb8b48bbb95e51196753eab9837ada Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Fri, 25 Nov 2022 13:59:01 -0800 Subject: [PATCH 7/9] make black --- bin/add_transform_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index c6e45c571..26cc7b58b 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -47,6 +47,7 @@ def add_regional_endpoint_configuration_if_needed(template: Dict[str, Any]) -> D return template + def replace_aws_partition(partition: str, file_path: str): template = read_json_file(file_path) with open(file_path, "w") as file: From 7d6e7d95c5a396b95d6c5e6b0d805bf1605e0e99 Mon Sep 17 00:00:00 2001 From: Xia Zhao Date: Fri, 25 Nov 2022 14:04:03 -0800 Subject: [PATCH 8/9] make pr --- bin/add_transform_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/add_transform_test.py b/bin/add_transform_test.py index 26cc7b58b..bb1457955 100644 --- a/bin/add_transform_test.py +++ b/bin/add_transform_test.py @@ -48,7 +48,7 @@ def add_regional_endpoint_configuration_if_needed(template: Dict[str, Any]) -> D return template -def replace_aws_partition(partition: str, file_path: str): +def replace_aws_partition(partition: str, file_path: str) -> None: template = read_json_file(file_path) with open(file_path, "w") as file: updated_template = json.loads(json.dumps(template).replace("arn:aws", f"aws:{partition}")) From 764527a509218d464a7d496b41baa4eced8ebb76 Mon Sep 17 00:00:00 2001 From: Xia Zhao <78883180+xazhao@users.noreply.github.com> Date: Fri, 25 Nov 2022 14:04:50 -0800 Subject: [PATCH 9/9] Update DEVELOPMENT_GUIDE.md Co-authored-by: Chris Rehn <1280602+hoffa@users.noreply.github.com> --- DEVELOPMENT_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPMENT_GUIDE.md b/DEVELOPMENT_GUIDE.md index 11677b93b..a6f5aed4f 100644 --- a/DEVELOPMENT_GUIDE.md +++ b/DEVELOPMENT_GUIDE.md @@ -143,7 +143,7 @@ To disable this feature, run the following command instead. python3 bin/add_transform_test.py --template-file template.yaml --disable-api-configuration ``` -The script will update the partition of arn based on the output path. To disable updating the partition of arn +The script automatically updates hardcoded ARN partitions to match the output partition. To disable this, use: ```bash python3 bin/add_transform_test.py --template-file template.yaml --disable-update-partition ```