Merge branch 'develop' into tmp/1678476371/main

.cfnlintrc.yaml

@@ -125,3 +125,4 @@ ignore_checks:
   - E2531 # Deprecated runtime; not relevant for transform tests
   - W2531 # EOL runtime; not relevant for transform tests
   - E3001 # Invalid or unsupported Type; common in transform tests since they focus on SAM resources
+  - W2001 # Parameter not used

.github/workflows/build.yml

@@ -22,6 +22,7 @@ jobs:
           - "3.8"
           - "3.9"
           - "3.10"
+          - "3.11"
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4

DEVELOPMENT_GUIDE.md

@@ -59,7 +59,7 @@ eval "$(pyenv virtualenv-init -)"
 We format our code using [Black](https://github.com/python/black) and verify the source code is black compliant
 during PR checks. Black will be installed automatically with `make init`.
 
-After installing, you can run our formatting through our Makefile by `make black` or integrating Black directly in your favorite IDE (instructions
+After installing, you can run our formatting through our Makefile by `make format` or integrating Black directly in your favorite IDE (instructions
 can be found [here](https://black.readthedocs.io/en/stable/editor_integration.html))
 
 ##### (Workaround) Integrating Black directly in your favorite IDE

Makefile

@@ -17,23 +17,37 @@ test-cov-report:
 integ-test:
 	pytest --no-cov integration/
 
-black:
+format:
 	black setup.py samtranslator tests integration bin schema_source
+	bin/transform-test-error-json-format.py --write tests/translator/output/error_*.json
 	bin/json-format.py --write tests integration samtranslator/policy_templates_data
 	bin/yaml-format.py --write tests
 	bin/yaml-format.py --write integration --add-test-metadata
 
-black-check:
+black:
+	$(warning `make black` is deprecated, please use `make format`)
+	# sleep for 5 seconds so the message can be seen.
+	sleep 5
+	make format
+
+format-check:
 	# Checking latest schema was generated (run `make schema` if this fails)
 	mkdir -p .tmp
 	python -m samtranslator.internal.schema_source.schema --sam-schema .tmp/sam.schema.json --cfn-schema schema_source/cloudformation.schema.json --unified-schema .tmp/schema.json
 	diff -u schema_source/sam.schema.json .tmp/sam.schema.json
 	diff -u samtranslator/schema/schema.json .tmp/schema.json
 	black --check setup.py samtranslator tests integration bin schema_source
+	bin/transform-test-error-json-format.py --check tests/translator/output/error_*.json
 	bin/json-format.py --check tests integration samtranslator/policy_templates_data
 	bin/yaml-format.py --check tests
 	bin/yaml-format.py --check integration --add-test-metadata
 
+black-check:
+	$(warning `make black-check` is deprecated, please use `make format-check`)
+	# sleep for 5 seconds so the message can be seen.
+	sleep 5
+	make format-check
+
 lint:
 	ruff samtranslator bin schema_source integration tests
 	# mypy performs type check
@@ -76,7 +90,7 @@ schema-all: fetch-schema-data update-schema-data schema
 dev: test
 
 # Verifications to run before sending a pull request
-pr: black-check lint init dev
+pr: format-check lint init dev
 
 clean:
 	rm -rf .tmp

bin/add_transform_test.py

@@ -6,13 +6,22 @@
 import shutil
 import subprocess
 import sys
-import tempfile
+from copy import deepcopy
 from pathlib import Path
 from typing import Any, Dict
 
+import boto3
+
+from samtranslator.translator.arn_generator import ArnGenerator
+from samtranslator.translator.managed_policy_translator import ManagedPolicyLoader
+from samtranslator.translator.transform import transform
+from samtranslator.yaml_helper import yaml_parse
+
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 TRANSFORM_TEST_DIR = os.path.join(SCRIPT_DIR, "..", "tests", "translator")
 
+iam_client = boto3.client("iam")
+
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument(
     "--template-file",
@@ -40,7 +49,7 @@ def read_json_file(file_path: str) -> Dict[str, Any]:
 
 def write_json_file(obj: Dict[str, Any], file_path: str) -> None:
     with open(file_path, "w", encoding="utf-8") as f:
-        json.dump(obj, f, indent=2)
+        json.dump(obj, f, indent=2, sort_keys=True)
 
 
 def add_regional_endpoint_configuration_if_needed(template: Dict[str, Any]) -> Dict[str, Any]:
@@ -66,38 +75,29 @@ def replace_aws_partition(partition: str, file_path: str) -> None:
 def generate_transform_test_output_files(input_file_path: str, file_basename: str) -> None:
     output_file_option = file_basename + ".json"
 
-    # run sam-translate.py and get the temporary output file
-    with tempfile.NamedTemporaryFile() as temp_output_file:
-        subprocess.run(
-            [
-                sys.executable,
-                os.path.join(SCRIPT_DIR, "sam-translate.py"),
-                "--template-file",
-                input_file_path,
-                "--output-template",
-                temp_output_file.name,
-            ],
-            check=True,
-        )
+    with open(os.path.join(input_file_path), "r") as f:
+        manifest = yaml_parse(f)  # type: ignore[no-untyped-call]
+
+    transform_test_output_paths = {
+        "aws": ("us-west-2", os.path.join(TRANSFORM_TEST_DIR, "output", output_file_option)),
+        "aws-cn": ("cn-north-1 ", os.path.join(TRANSFORM_TEST_DIR, "output/aws-cn/", output_file_option)),
+        "aws-us-gov": ("us-gov-west-1", os.path.join(TRANSFORM_TEST_DIR, "output/aws-us-gov/", output_file_option)),
+    }
 
-        # copy the output files into correct directories
-        transform_test_output_path = os.path.join(TRANSFORM_TEST_DIR, "output", output_file_option)
-        shutil.copyfile(temp_output_file.name, transform_test_output_path)
+    for partition, (region, output_path) in transform_test_output_paths.items():
+        # Set Boto Session Region to guarantee the same hash input as transform tests for API deployment id
+        ArnGenerator.BOTO_SESSION_REGION_NAME = region
+        # Implicit API Plugin may alter input template file, thus passing a copy here.
+        output_fragment = transform(deepcopy(manifest), {}, ManagedPolicyLoader(iam_client))
 
-        regional_transform_test_output_paths = {
-            "aws-cn": os.path.join(TRANSFORM_TEST_DIR, "output/aws-cn/", output_file_option),
-            "aws-us-gov": os.path.join(TRANSFORM_TEST_DIR, "output/aws-us-gov/", output_file_option),
-        }
+        if not CLI_OPTIONS.disable_api_configuration and partition != "aws":
+            output_fragment = add_regional_endpoint_configuration_if_needed(output_fragment)
 
-        if not CLI_OPTIONS.disable_api_configuration:
-            template = read_json_file(temp_output_file.name)
-            template = add_regional_endpoint_configuration_if_needed(template)
-            write_json_file(template, temp_output_file.name)
+        write_json_file(output_fragment, output_path)
 
-        for partition, output_path in regional_transform_test_output_paths.items():
-            shutil.copyfile(temp_output_file.name, output_path)
-            if not CLI_OPTIONS.disable_update_partition:
-                replace_aws_partition(partition, output_path)
+        # Update arn partition if necessary
+        if not CLI_OPTIONS.disable_update_partition:
+            replace_aws_partition(partition, output_path)
 
 
 def get_input_file_path() -> str:
@@ -118,6 +118,18 @@ def verify_input_template(input_file_path: str):  # type: ignore[no-untyped-def]
         )
 
 
+def format_test_files() -> None:
+    subprocess.run(
+        [sys.executable, os.path.join(SCRIPT_DIR, "json-format.py"), "--write", "tests"],
+        check=True,
+    )
+
+    subprocess.run(
+        [sys.executable, os.path.join(SCRIPT_DIR, "yaml-format.py"), "--write", "tests"],
+        check=True,
+    )
+
+
 def main() -> None:
     input_file_path = get_input_file_path()
     file_basename = Path(input_file_path).stem
@@ -133,6 +145,8 @@ def main() -> None:
         "Generating transform test input and output files complete. \n\nPlease check the generated output is as expected. This tool does not guarantee correct output."
     )
 
+    format_test_files()
+
 
 if __name__ == "__main__":
     main()

bin/parse_docs.py

@@ -20,7 +20,7 @@
 def parse(s: str) -> Iterator[Tuple[str, str]]:
     """Parse an AWS docs page in Markdown format, yielding each property."""
     # Prevent from parsing return values accidentally
-    s = stringbetween(s, "#\s+Properties", "#\s+Return values")
+    s = stringbetween(s, "#\\s+Properties", "#\\s+Return values")
     if not s:
         return
     parts = s.split("\n\n")
@@ -44,7 +44,7 @@ def remove_first_line(s: str) -> str:
 
 
 def convert_to_full_path(description: str, prefix: str) -> str:
-    pattern = re.compile("\(([#\.a-zA-Z0-9_-]+)\)")
+    pattern = re.compile("\\(([#\\.a-zA-Z0-9_-]+)\\)")
     matched_content = pattern.findall(description)
 
     for path in matched_content:

bin/public_interface.py

@@ -161,10 +161,8 @@ def _only_new_optional_arguments_or_existing_arguments_optionalized_or_var_argum
         return False
     # it is an optional argument when it has a default value:
     return all(
-        [
-            "default" in argument or argument["kind"] in ("VAR_KEYWORD", "VAR_POSITIONAL")
-            for argument in arguments[len(original_arguments) :]
-        ]
+        "default" in argument or argument["kind"] in ("VAR_KEYWORD", "VAR_POSITIONAL")
+        for argument in arguments[len(original_arguments) :]
     )
 
 

bin/sam-translate.py

@@ -40,7 +40,7 @@
     "--output-template",
     help="Location to store resulting CloudFormation template [default: transformed-template.json].",
     type=Path,
-    default=Path("transformed-template.yaml"),
+    default=Path("transformed-template.json"),
 )
 parser.add_argument(
     "--s3-bucket",

bin/transform-test-error-json-format.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Transform test error JSON file formatter (without prettier).
+
+It makes error json easier to review by breaking down "errorMessage"
+into list of strings (delimiter: ". ").
+"""
+import os
+import sys
+
+from typing_extensions import Final
+
+my_path = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, my_path + "/..")
+
+import json
+from typing import Type
+
+from bin._file_formatter import FileFormatter
+
+
+class TransformTestErrorJSONFormatter(FileFormatter):
+    _ERROR_MESSAGE_KEY: Final[str] = "errorMessage"
+    _BREAKDOWN_ERROR_MESSAGE_KEY: Final[str] = "_autoGeneratedBreakdownErrorMessage"
+    _DELIMITER: Final[str] = ". "
+
+    @staticmethod
+    def description() -> str:
+        return "Transform test error JSON file formatter"
+
+    def format_str(self, input_str: str) -> str:
+        """
+        It makes error json easier to review by breaking down "errorMessage"
+        into list of strings (delimiter: ". ").
+        """
+        obj = json.loads(input_str)
+        error_message = obj.get(self._ERROR_MESSAGE_KEY)
+        if isinstance(error_message, str):
+            tokens = error_message.split(self._DELIMITER)
+            obj[self._BREAKDOWN_ERROR_MESSAGE_KEY] = [
+                token if index == len(tokens) - 1 else token + self._DELIMITER for index, token in enumerate(tokens)
+            ]
+        return json.dumps(obj, indent=2, sort_keys=True) + "\n"
+
+    @staticmethod
+    def decode_exception() -> Type[Exception]:
+        return json.JSONDecodeError
+
+    @staticmethod
+    def file_extension() -> str:
+        return ".json"
+
+
+if __name__ == "__main__":
+    TransformTestErrorJSONFormatter.main()

integration/combination/test_function_with_sns.py

@@ -51,3 +51,4 @@ def test_function_with_sns_intrinsics(self):
         subscription_arn = subscription["SubscriptionArn"]
         subscription_attributes = sns_client.get_subscription_attributes(SubscriptionArn=subscription_arn)
         self.assertEqual(subscription_attributes["Attributes"]["FilterPolicy"], '{"price_usd":[{"numeric":["<",100]}]}')
+        self.assertEqual(subscription_attributes["Attributes"]["FilterPolicyScope"], "MessageAttributes")

integration/resources/templates/combination/function_with_sns_intrinsics.yaml

@@ -30,6 +30,7 @@ Resources:
                 - numeric:
                   - <
                   - 100
+            FilterPolicyScope: MessageAttributes
             Region:
               Ref: AWS::Region
             SqsSubscription: true

requirements/dev.txt

@@ -4,7 +4,7 @@ pytest-xdist>=2.5,<4
 pytest-env>=0.6,<1
 pytest-rerunfailures>=9.1,<12
 pyyaml~=6.0
-ruff==0.0.252  # loose the requirement once it is more stable
+ruff==0.0.253  # loose the requirement once it is more stable
 
 # Test requirements
 pytest>=6.2,<8

ruff.toml

@@ -2,7 +2,8 @@
 line-length = 999
 
 select = [
-    "E",  # Pyflakes
+    "E",  # pycodestyle
+    "W",  # pycodestyle
     "F",  # Pyflakes
     "PL", # pylint
     "I", # isort
@@ -17,6 +18,7 @@ select = [
     "SIM", # flake8-simplify
     "TID", # flake8-tidy-imports
     "RUF", # Ruff-specific rules
+    "YTT", # flake8-2020
 ]
 
 # Mininal python version we support is 3.7

samtranslator/feature_toggle/dialup.py

@@ -5,7 +5,7 @@
 class BaseDialup(ABC):
     """BaseDialup class to provide an interface for all dialup classes"""
 
-    def __init__(self, region_config, **kwargs):  # type: ignore[no-untyped-def]
+    def __init__(self, region_config, **kwargs) -> None:  # type: ignore[no-untyped-def]
         self.region_config = region_config
 
     @abstractmethod
@@ -23,8 +23,8 @@ class DisabledDialup(BaseDialup):
     A dialup that is never enabled
     """
 
-    def __init__(self, region_config, **kwargs):  # type: ignore[no-untyped-def]
-        super().__init__(region_config)  # type: ignore[no-untyped-call]
+    def __init__(self, region_config, **kwargs) -> None:  # type: ignore[no-untyped-def]
+        super().__init__(region_config)
 
     def is_enabled(self) -> bool:
         return False
@@ -36,8 +36,8 @@ class ToggleDialup(BaseDialup):
     Example of region_config: { "type": "toggle", "enabled": True }
     """
 
-    def __init__(self, region_config, **kwargs):  # type: ignore[no-untyped-def]
-        super().__init__(region_config)  # type: ignore[no-untyped-call]
+    def __init__(self, region_config, **kwargs) -> None:  # type: ignore[no-untyped-def]
+        super().__init__(region_config)
         self.region_config = region_config
 
     def is_enabled(self):  # type: ignore[no-untyped-def]
@@ -50,8 +50,8 @@ class SimpleAccountPercentileDialup(BaseDialup):
     Example of region_config: { "type": "account-percentile", "enabled-%": 20 }
     """
 
-    def __init__(self, region_config, account_id, feature_name, **kwargs):  # type: ignore[no-untyped-def]
-        super().__init__(region_config)  # type: ignore[no-untyped-call]
+    def __init__(self, region_config, account_id, feature_name, **kwargs) -> None:  # type: ignore[no-untyped-def]
+        super().__init__(region_config)
         self.account_id = account_id
         self.feature_name = feature_name
 

samtranslator/feature_toggle/feature_toggle.py

@@ -56,7 +56,7 @@ def _get_dialup(self, region_config, feature_name):  # type: ignore[no-untyped-d
                 region_config, account_id=self.account_id, feature_name=feature_name
             )
         LOG.warning("Dialup type '{}' is None or is not supported.".format(dialup_type))
-        return DisabledDialup(region_config)  # type: ignore[no-untyped-call]
+        return DisabledDialup(region_config)
 
     def is_enabled(self, feature_name: str) -> bool:
         """
@@ -119,7 +119,7 @@ def config(self) -> Dict[str, Any]:
 class FeatureToggleLocalConfigProvider(FeatureToggleConfigProvider):
     """Feature toggle config provider which uses a local file. This is to facilitate local testing."""
 
-    def __init__(self, local_config_path):  # type: ignore[no-untyped-def]
+    def __init__(self, local_config_path) -> None:  # type: ignore[no-untyped-def]
         FeatureToggleConfigProvider.__init__(self)
         with open(local_config_path, "r", encoding="utf-8") as f:
             config_json = f.read()
@@ -134,7 +134,7 @@ class FeatureToggleAppConfigConfigProvider(FeatureToggleConfigProvider):
     """Feature toggle config provider which loads config from AppConfig."""
 
     @cw_timer(prefix="External", name="AppConfig")
-    def __init__(self, application_id, environment_id, configuration_profile_id, app_config_client=None):  # type: ignore[no-untyped-def]
+    def __init__(self, application_id, environment_id, configuration_profile_id, app_config_client=None) -> None:  # type: ignore[no-untyped-def]
         FeatureToggleConfigProvider.__init__(self)
         try:
             LOG.info("Loading feature toggle config from AppConfig...")