From d262e0396f3cadbc4cdddad5cf7940dce147b62d Mon Sep 17 00:00:00 2001 From: Chris Rehn <1280602+hoffa@users.noreply.github.com> Date: Thu, 29 Sep 2022 14:50:43 -0700 Subject: [PATCH 01/11] Verify profile variables have been replaced (#264) --- .../model/connector_profiles/profile.py | 10 +++++++++ samtranslator/model/sam_resources.py | 3 +++ .../model/connector_profiles/test_profile.py | 22 ++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/samtranslator/model/connector_profiles/profile.py b/samtranslator/model/connector_profiles/profile.py index 65b6f3a14..556fd61c1 100644 --- a/samtranslator/model/connector_profiles/profile.py +++ b/samtranslator/model/connector_profiles/profile.py @@ -13,6 +13,16 @@ def get_profile(source_type: str, dest_type: str): return PROFILE["Permissions"].get(source_type, {}).get(dest_type) +def verify_profile_variables_replaced(obj: Any) -> None: + """ + Verifies all profile variables have been replaced; throws ValueError if not. + """ + s = json.dumps(obj) + matches = re.findall(r"%{[\w\.]+}", s) + if matches: + raise ValueError(f"The following variables have not been replaced: {matches}") + + def profile_replace(obj: Any, replacements: Dict[str, Any]): """ This function is used to recursively replace all keys in 'replacements' found diff --git a/samtranslator/model/sam_resources.py b/samtranslator/model/sam_resources.py index 0a7dd3102..c6864ad53 100644 --- a/samtranslator/model/sam_resources.py +++ b/samtranslator/model/sam_resources.py @@ -13,6 +13,7 @@ ConnectorProfile, profile_replace, get_profile, + verify_profile_variables_replaced, ) import samtranslator.model.eventsources @@ -1694,6 +1695,8 @@ def to_cloudformation(self, **kwargs) -> List: except ValueError as e: raise InvalidResourceException(self.logical_id, str(e)) + verify_profile_variables_replaced(profile_properties) + generated_resources: List[Resource] = [] if profile_type == "AWS_IAM_ROLE_MANAGED_POLICY": generated_resources.append( diff --git a/tests/model/connector_profiles/test_profile.py b/tests/model/connector_profiles/test_profile.py index 5bc66397d..e621e2c52 100644 --- a/tests/model/connector_profiles/test_profile.py +++ b/tests/model/connector_profiles/test_profile.py @@ -1,6 +1,8 @@ from unittest import TestCase -from samtranslator.model.connector_profiles.profile import profile_replace +from parameterized import parameterized + +from samtranslator.model.connector_profiles.profile import profile_replace, verify_profile_variables_replaced class TestProfile(TestCase): @@ -215,3 +217,21 @@ def test_profile_replace_dict_input(self): } }, ) + + def test_verify_replaced(self): + verify_profile_variables_replaced({"Foo": {"Bar": "${AllGood}something What"}}) + verify_profile_variables_replaced({"Foo": {"Bar": "${All.Good}something %{What"}}) + verify_profile_variables_replaced({"Foo": {"${AllGood}": "something %{What"}}) + + @parameterized.expand( + [ + ({"Foo": {"Bar": "%{NotGood}something What"}}, "%{NotGood}"), + ({"Foo": {"Bar": "%{Not.Good}something What"}}, "%{Not.Good}"), + ({"Foo": {"%{NotGood}": "something What"}}, "%{NotGood}"), + ({"Foo": {"%{NotGood}": "something %{What.No}"}}, "['%{NotGood}', '%{What.No}']"), + ] + ) + def test_verify_not_replaced(self, profile, error_includes): + with self.assertRaises(ValueError) as ctx: + verify_profile_variables_replaced(profile) + self.assertIn(error_includes, str(ctx.exception)) From 848299dedc0ca5c0d0021e60dc7d190f8ac327c3 Mon Sep 17 00:00:00 2001 From: hnnasit <84355507+hnnasit@users.noreply.github.com> Date: Thu, 29 Sep 2022 17:34:08 -0700 Subject: [PATCH 02/11] Fix integ tests for feature toggle related to custom domain (#266) --- .../test_custom_http_api_domains_test.py | 5 ++++- .../test_custom_rest_api_domains.py | 12 ++++++++++-- integration/helpers/base_test.py | 18 ++++++++++++++++-- ...ith_custom_domains_edge_feature_toggle.json | 11 +++++++++++ ...custom_domains_regional_feature_toggle.json | 13 +++++++++++++ ..._ownership_verification_feature_toggle.json | 13 +++++++++++++ ...custom_domains_regional_feature_toggle.json | 12 ++++++++++++ ..._ownership_verification_feature_toggle.json | 12 ++++++++++++ .../api_with_custom_domains_edge.yaml | 5 ++++- .../api_with_custom_domains_regional.yaml | 5 ++++- ...omains_regional_ownership_verification.yaml | 5 ++++- .../http_api_with_custom_domains_regional.yaml | 5 ++++- ...omains_regional_ownership_verification.yaml | 5 ++++- 13 files changed, 111 insertions(+), 10 deletions(-) create mode 100644 integration/resources/expected/combination/api_with_custom_domains_edge_feature_toggle.json create mode 100644 integration/resources/expected/combination/api_with_custom_domains_regional_feature_toggle.json create mode 100644 integration/resources/expected/combination/api_with_custom_domains_regional_ownership_verification_feature_toggle.json create mode 100644 integration/resources/expected/combination/http_api_with_custom_domains_regional_feature_toggle.json create mode 100644 integration/resources/expected/combination/http_api_with_custom_domains_regional_ownership_verification_feature_toggle.json diff --git a/integration/combination/test_custom_http_api_domains_test.py b/integration/combination/test_custom_http_api_domains_test.py index 6aa66623d..fbff683e9 100644 --- a/integration/combination/test_custom_http_api_domains_test.py +++ b/integration/combination/test_custom_http_api_domains_test.py @@ -21,7 +21,10 @@ def test_custom_http_api_domains_regional(self): api_gateway_client = self.client_provider.api_v2_client result = api_gateway_client.get_domain_name(DomainName=domain_name_id) - self.assertEqual("httpapi.sam-gamma-regional.com", result["DomainName"]) + if "FeatureToggle" in self.pipeline_prefix: + self.assertEqual("httpapi.ftl.sam-gamma-regional.com", result["DomainName"]) + else: + self.assertEqual("httpapi.sam-gamma-regional.com", result["DomainName"]) mtls_auth_config = result["MutualTlsAuthentication"] self.assertEqual(self.file_to_s3_uri_map["MTLSCert.pem"]["uri"], mtls_auth_config["TruststoreUri"]) diff --git a/integration/combination/test_custom_rest_api_domains.py b/integration/combination/test_custom_rest_api_domains.py index f5319723a..83e220fa8 100644 --- a/integration/combination/test_custom_rest_api_domains.py +++ b/integration/combination/test_custom_rest_api_domains.py @@ -12,6 +12,7 @@ class TestCustomRestApiDomains(BaseInternalTest): def test_custom_rest_api_domains_edge(self): self.create_and_verify_stack("combination/api_with_custom_domains_edge") + domain_name_list = self.get_stack_resources("AWS::ApiGateway::DomainName") self.assertEqual(1, len(domain_name_list)) @@ -19,7 +20,10 @@ def test_custom_rest_api_domains_edge(self): api_gateway_client = self.client_provider.api_client result = api_gateway_client.get_domain_name(domainName=domain_name_id) - self.assertEqual("sam-gamma-edge.com", result["domainName"]) + if "FeatureToggle" in self.pipeline_prefix: + self.assertEqual("ftl.sam-gamma-edge.com", result["domainName"]) + else: + self.assertEqual("sam-gamma-edge.com", result["domainName"]) end_point_config = result["endpointConfiguration"] end_point_types = end_point_config["types"] @@ -37,7 +41,11 @@ def test_custom_rest_api_domains_regional(self): api_gateway_client = self.client_provider.api_client result = api_gateway_client.get_domain_name(domainName=domain_name_id) - self.assertEqual("sam-gamma-regional.com", result["domainName"]) + if "FeatureToggle" in self.pipeline_prefix: + self.assertEqual("ftl.sam-gamma-regional.com", result["domainName"]) + else: + self.assertEqual("sam-gamma-regional.com", result["domainName"]) + self.assertEqual("TLS_1_2", result["securityPolicy"]) end_point_config = result["endpointConfiguration"] diff --git a/integration/helpers/base_test.py b/integration/helpers/base_test.py index a61a1c3e7..cce397eb7 100644 --- a/integration/helpers/base_test.py +++ b/integration/helpers/base_test.py @@ -51,6 +51,13 @@ STACK_NAME_PREFIX = "sam-integ-stack-" S3_BUCKET_PREFIX = "sam-integ-bucket-" +FEATURE_TOGGLE_JSON_FILES = [ + "http_api_with_custom_domains_regional", + "http_api_with_custom_domains_regional_ownership_verification", + "api_with_custom_domains_edge", + "api_with_custom_domains_regional", + "api_with_custom_domains_regional_ownership_verification", +] class BaseTest(TestCase): @@ -154,13 +161,20 @@ def create_and_verify_stack(self, file_path, parameters=None, s3_uploader=None): s3_uploader: S3Uploader object Object for uploading files to s3 """ - folder, file_name = file_path.split("/") # If template is too large, calling the method with self.s3_uploader to send the template to s3 then deploy self.create_stack(file_path, parameters, s3_uploader) - self.expected_resource_path = str(Path(self.expected_dir, folder, file_name + ".json")) + self.get_expected_json_file_name(file_path) self.verify_stack() + def get_expected_json_file_name(self, file_path): + folder, file_name = file_path.split("/") + + if "FeatureToggle" in self.pipeline_prefix and file_name in FEATURE_TOGGLE_JSON_FILES: + self.expected_resource_path = str(Path(self.expected_dir, folder, file_name + "_feature_toggle.json")) + else: + self.expected_resource_path = str(Path(self.expected_dir, folder, file_name + ".json")) + def skip_using_service_detector(self, file_path): """ Skips the test if it cannot pass the test of diff --git a/integration/resources/expected/combination/api_with_custom_domains_edge_feature_toggle.json b/integration/resources/expected/combination/api_with_custom_domains_edge_feature_toggle.json new file mode 100644 index 000000000..83afc8d73 --- /dev/null +++ b/integration/resources/expected/combination/api_with_custom_domains_edge_feature_toggle.json @@ -0,0 +1,11 @@ +[ + { "LogicalResourceId":"RecordSetGroupc911be5759", "ResourceType":"AWS::Route53::RecordSetGroup" }, + { "LogicalResourceId":"MyApiDeployment", "ResourceType":"AWS::ApiGateway::Deployment" }, + { "LogicalResourceId":"ApiGatewayDomainName299fac327d", "ResourceType":"AWS::ApiGateway::DomainName" }, + { "LogicalResourceId":"MyApi", "ResourceType":"AWS::ApiGateway::RestApi" }, + { "LogicalResourceId":"MyApiProdStage", "ResourceType":"AWS::ApiGateway::Stage" }, + { "LogicalResourceId":"MyFunction", "ResourceType":"AWS::Lambda::Function" }, + { "LogicalResourceId":"MyFunctionRole", "ResourceType":"AWS::IAM::Role" }, + { "LogicalResourceId":"MyApigetBasePathMapping", "ResourceType":"AWS::ApiGateway::BasePathMapping" }, + { "LogicalResourceId":"MyFunctionFetchPermissionProd", "ResourceType":"AWS::Lambda::Permission" } + ] \ No newline at end of file diff --git a/integration/resources/expected/combination/api_with_custom_domains_regional_feature_toggle.json b/integration/resources/expected/combination/api_with_custom_domains_regional_feature_toggle.json new file mode 100644 index 000000000..d83cd1c29 --- /dev/null +++ b/integration/resources/expected/combination/api_with_custom_domains_regional_feature_toggle.json @@ -0,0 +1,13 @@ +[ + { "LogicalResourceId":"MyFunctionImplicitGetPermissionProd", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"ServerlessRestApiDeployment", "ResourceType":"AWS::ApiGateway::Deployment" }, + { "LogicalResourceId":"ServerlessRestApipostBasePathMapping", "ResourceType":"AWS::ApiGateway::BasePathMapping" }, + { "LogicalResourceId":"ServerlessRestApi", "ResourceType":"AWS::ApiGateway::RestApi" }, + { "LogicalResourceId":"RecordSetGroupd17dced08c", "ResourceType":"AWS::Route53::RecordSetGroup" }, + { "LogicalResourceId":"ServerlessRestApiProdStage", "ResourceType":"AWS::ApiGateway::Stage" }, + { "LogicalResourceId":"ServerlessRestApigetBasePathMapping", "ResourceType":"AWS::ApiGateway::BasePathMapping" }, + { "LogicalResourceId":"ApiGatewayDomainName98c928338d", "ResourceType":"AWS::ApiGateway::DomainName" }, + { "LogicalResourceId":"MyFunction", "ResourceType":"AWS::Lambda::Function" }, + { "LogicalResourceId":"MyFunctionImplicitPostPermissionProd", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"MyFunctionRole", "ResourceType":"AWS::IAM::Role" } + ] \ No newline at end of file diff --git a/integration/resources/expected/combination/api_with_custom_domains_regional_ownership_verification_feature_toggle.json b/integration/resources/expected/combination/api_with_custom_domains_regional_ownership_verification_feature_toggle.json new file mode 100644 index 000000000..815a864c0 --- /dev/null +++ b/integration/resources/expected/combination/api_with_custom_domains_regional_ownership_verification_feature_toggle.json @@ -0,0 +1,13 @@ +[ + { "LogicalResourceId":"MyFunctionImplicitGetPermissionProd", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"ServerlessRestApiDeployment", "ResourceType":"AWS::ApiGateway::Deployment" }, + { "LogicalResourceId":"ServerlessRestApipostBasePathMapping", "ResourceType":"AWS::ApiGateway::BasePathMapping" }, + { "LogicalResourceId":"ServerlessRestApi", "ResourceType":"AWS::ApiGateway::RestApi" }, + { "LogicalResourceId":"RecordSetGroupd17dced08c", "ResourceType":"AWS::Route53::RecordSetGroup" }, + { "LogicalResourceId":"ServerlessRestApiProdStage", "ResourceType":"AWS::ApiGateway::Stage" }, + { "LogicalResourceId":"ServerlessRestApigetBasePathMapping", "ResourceType":"AWS::ApiGateway::BasePathMapping" }, + { "LogicalResourceId":"ApiGatewayDomainNamef593820b0b", "ResourceType":"AWS::ApiGateway::DomainName" }, + { "LogicalResourceId":"MyFunction", "ResourceType":"AWS::Lambda::Function" }, + { "LogicalResourceId":"MyFunctionImplicitPostPermissionProd", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"MyFunctionRole", "ResourceType":"AWS::IAM::Role" } + ] \ No newline at end of file diff --git a/integration/resources/expected/combination/http_api_with_custom_domains_regional_feature_toggle.json b/integration/resources/expected/combination/http_api_with_custom_domains_regional_feature_toggle.json new file mode 100644 index 000000000..9808fb950 --- /dev/null +++ b/integration/resources/expected/combination/http_api_with_custom_domains_regional_feature_toggle.json @@ -0,0 +1,12 @@ +[ + { "LogicalResourceId":"MyFunctionImplicitGetPermission", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"MyFunctionImplicitPostPermission", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"MyApipostApiMapping", "ResourceType":"AWS::ApiGatewayV2::ApiMapping" }, + { "LogicalResourceId":"MyApigetApiMapping", "ResourceType":"AWS::ApiGatewayV2::ApiMapping" }, + { "LogicalResourceId":"MyApi", "ResourceType":"AWS::ApiGatewayV2::Api" }, + { "LogicalResourceId":"RecordSetGroupd17dced08c", "ResourceType":"AWS::Route53::RecordSetGroup" }, + { "LogicalResourceId":"MyApiProdStage", "ResourceType":"AWS::ApiGatewayV2::Stage" }, + { "LogicalResourceId":"ApiGatewayDomainNameV26198c55d75", "ResourceType":"AWS::ApiGatewayV2::DomainName" }, + { "LogicalResourceId":"MyFunction", "ResourceType":"AWS::Lambda::Function" }, + { "LogicalResourceId":"MyFunctionRole", "ResourceType":"AWS::IAM::Role" } + ] \ No newline at end of file diff --git a/integration/resources/expected/combination/http_api_with_custom_domains_regional_ownership_verification_feature_toggle.json b/integration/resources/expected/combination/http_api_with_custom_domains_regional_ownership_verification_feature_toggle.json new file mode 100644 index 000000000..3e37f7b6a --- /dev/null +++ b/integration/resources/expected/combination/http_api_with_custom_domains_regional_ownership_verification_feature_toggle.json @@ -0,0 +1,12 @@ +[ + { "LogicalResourceId":"MyFunctionImplicitGetPermission", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"MyFunctionImplicitPostPermission", "ResourceType":"AWS::Lambda::Permission" }, + { "LogicalResourceId":"MyApipostApiMapping", "ResourceType":"AWS::ApiGatewayV2::ApiMapping" }, + { "LogicalResourceId":"MyApigetApiMapping", "ResourceType":"AWS::ApiGatewayV2::ApiMapping" }, + { "LogicalResourceId":"MyApi", "ResourceType":"AWS::ApiGatewayV2::Api" }, + { "LogicalResourceId":"RecordSetGroupd17dced08c", "ResourceType":"AWS::Route53::RecordSetGroup" }, + { "LogicalResourceId":"MyApiProdStage", "ResourceType":"AWS::ApiGatewayV2::Stage" }, + { "LogicalResourceId":"ApiGatewayDomainNameV2483cac8ea6", "ResourceType":"AWS::ApiGatewayV2::DomainName" }, + { "LogicalResourceId":"MyFunction", "ResourceType":"AWS::Lambda::Function" }, + { "LogicalResourceId":"MyFunctionRole", "ResourceType":"AWS::IAM::Role" } + ] \ No newline at end of file diff --git a/integration/resources/templates/combination/api_with_custom_domains_edge.yaml b/integration/resources/templates/combination/api_with_custom_domains_edge.yaml index 390329adf..582468f84 100644 --- a/integration/resources/templates/combination/api_with_custom_domains_edge.yaml +++ b/integration/resources/templates/combination/api_with_custom_domains_edge.yaml @@ -3,6 +3,8 @@ Parameters: Type: String MyEdgeDomainCert: Type: String + HostedZoneId: + Type: String Resources: MyFunction: @@ -41,5 +43,6 @@ Resources: BasePath: - /get Route53: - HostedZoneId: Z1SKZDMQ2UR7IW + HostedZoneId: + Ref: HostedZoneId IpV6: true \ No newline at end of file diff --git a/integration/resources/templates/combination/api_with_custom_domains_regional.yaml b/integration/resources/templates/combination/api_with_custom_domains_regional.yaml index b5b131ce7..779deb0e8 100644 --- a/integration/resources/templates/combination/api_with_custom_domains_regional.yaml +++ b/integration/resources/templates/combination/api_with_custom_domains_regional.yaml @@ -3,6 +3,8 @@ Parameters: Type: String MyRestRegionalDomainCert: Type: String + HostedZoneId: + Type: String Globals: Api: @@ -20,7 +22,8 @@ Globals: - /get - /post Route53: - HostedZoneId: Z1DTV8GVAVOHDR + HostedZoneId: + Ref: HostedZoneId Resources: MyFunction: diff --git a/integration/resources/templates/combination/api_with_custom_domains_regional_ownership_verification.yaml b/integration/resources/templates/combination/api_with_custom_domains_regional_ownership_verification.yaml index fb6ab03d4..1f73730a2 100644 --- a/integration/resources/templates/combination/api_with_custom_domains_regional_ownership_verification.yaml +++ b/integration/resources/templates/combination/api_with_custom_domains_regional_ownership_verification.yaml @@ -5,6 +5,8 @@ Parameters: Type: String MyDomainOwnershipVerificationCertificate: Type: String + HostedZoneId: + Type: String Globals: Api: @@ -22,7 +24,8 @@ Globals: - /get - /post Route53: - HostedZoneId: Z1DTV8GVAVOHDR + HostedZoneId: + Ref: HostedZoneId OwnershipVerificationCertificateArn: Ref: MyDomainOwnershipVerificationCertificate diff --git a/integration/resources/templates/combination/http_api_with_custom_domains_regional.yaml b/integration/resources/templates/combination/http_api_with_custom_domains_regional.yaml index a13c88bc6..5fceb14b0 100644 --- a/integration/resources/templates/combination/http_api_with_custom_domains_regional.yaml +++ b/integration/resources/templates/combination/http_api_with_custom_domains_regional.yaml @@ -3,6 +3,8 @@ Parameters: Type: String MyDomainCert: Type: String + HostedZoneId: + Type: String Globals: HttpApi: @@ -20,7 +22,8 @@ Globals: - /get - /post Route53: - HostedZoneId: Z1DTV8GVAVOHDR + HostedZoneId: + Ref: HostedZoneId Resources: MyFunction: diff --git a/integration/resources/templates/combination/http_api_with_custom_domains_regional_ownership_verification.yaml b/integration/resources/templates/combination/http_api_with_custom_domains_regional_ownership_verification.yaml index 3e18d5452..b8ffb2ba8 100644 --- a/integration/resources/templates/combination/http_api_with_custom_domains_regional_ownership_verification.yaml +++ b/integration/resources/templates/combination/http_api_with_custom_domains_regional_ownership_verification.yaml @@ -5,6 +5,8 @@ Parameters: Type: String MyOwnershipVerificationCert: Type: String + HostedZoneId: + Type: String Globals: HttpApi: @@ -22,7 +24,8 @@ Globals: - /get - /post Route53: - HostedZoneId: Z1DTV8GVAVOHDR + HostedZoneId: + Ref: HostedZoneId OwnershipVerificationCertificateArn: Ref: MyOwnershipVerificationCert From 2bd6135b735c76d837f8c0c3c39c635ed16f99d1 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Fri, 30 Sep 2022 10:08:22 -0700 Subject: [PATCH 03/11] feat: Event filtering for MQ, MSK, SMK (#134) (#265) Support event filtering for Amazon MQ, MSK and SelfManagedKafka event sources Co-authored-by: Renato Valenzuela <37676028+valerena@users.noreply.github.com> --- samtranslator/model/eventsources/pull.py | 2 +- .../input/error_event_filtering.yaml | 19 --- .../input/function_with_event_filtering.yaml | 42 ++++- .../aws-cn/function_with_event_filtering.json | 144 +++++++++++++++++- .../function_with_event_filtering.json | 144 +++++++++++++++++- .../output/error_event_filtering.json | 2 +- .../output/function_with_event_filtering.json | 144 +++++++++++++++++- 7 files changed, 472 insertions(+), 25 deletions(-) diff --git a/samtranslator/model/eventsources/pull.py b/samtranslator/model/eventsources/pull.py index 658395de3..b8df62a70 100644 --- a/samtranslator/model/eventsources/pull.py +++ b/samtranslator/model/eventsources/pull.py @@ -21,7 +21,7 @@ class PullEventSource(ResourceMacro): """ # Event types that support `FilterCriteria`, stored as a list to keep the alphabetical order - RESOURCE_TYPES_WITH_EVENT_FILTERING = ["DynamoDB", "Kinesis", "SQS"] + RESOURCE_TYPES_WITH_EVENT_FILTERING = ["DynamoDB", "Kinesis", "MQ", "MSK", "SelfManagedKafka", "SQS"] # Note(xinhol): `PullEventSource` should have been an abstract class. Disabling the type check for the next # line to avoid any potential behavior change. diff --git a/tests/translator/input/error_event_filtering.yaml b/tests/translator/input/error_event_filtering.yaml index ef575217f..a7fe532e8 100644 --- a/tests/translator/input/error_event_filtering.yaml +++ b/tests/translator/input/error_event_filtering.yaml @@ -15,25 +15,6 @@ Resources: FiltersToUse: - Pattern: '{"name": "value"}' - NotSupportedPullEvent: - Type: AWS::Serverless::Function - Properties: - CodeUri: s3://sam-demo-bucket/filtered_events.zip - Handler: index.handler - Runtime: nodejs16.x - Events: - KafkaEvent: - Type: MSK - Properties: - StartingPosition: LATEST - Stream: arn:aws:kafka:us-east-1:012345678012:cluster/clusterName/abcdefab-1234-abcd-5678-cdef0123ab01-2 - Topics: - - MyTopic - FilterCriteria: - Filters: - - Pattern: '{"name": "value"}' - - NotSupportedPushEvent: Type: AWS::Serverless::Function Properties: diff --git a/tests/translator/input/function_with_event_filtering.yaml b/tests/translator/input/function_with_event_filtering.yaml index 5aabd1614..32af43813 100644 --- a/tests/translator/input/function_with_event_filtering.yaml +++ b/tests/translator/input/function_with_event_filtering.yaml @@ -29,13 +29,53 @@ Resources: } } }' - MySqsQueue: + MySqsEvent: Type: SQS Properties: Queue: !GetAtt MySqsQueue.Arn FilterCriteria: Filters: - Pattern: '{"name": "value"}' + MSKEvent: + Type: MSK + Properties: + StartingPosition: LATEST + Stream: arn:aws:kafka:us-west-2:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2 + Topics: + - "MyDummyTestTopic" + FilterCriteria: + Filters: + - Pattern: '{"name": "value"}' + MyKafkaEvent: + Type: SelfManagedKafka + Properties: + KafkaBootstrapServers: + - "abc.xyz.com:9092" + Topics: + - "Topic1" + SourceAccessConfigurations: + - Type: SASL_SCRAM_512_AUTH + URI: arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c + - Type: VPC_SUBNET + URI: subnet:subnet-12345 + - Type: VPC_SECURITY_GROUP + URI: security_group:sg-67890 + FilterCriteria: + Filters: + - Pattern: '{"name": "value"}' + MyMQQueue: + Type: MQ + Properties: + Broker: arn:aws:mq:us-east-2:123456789012:broker:MyBroker:b-1234a5b6-78cd-901e-2fgh-3i45j6k178l9 + Queues: + - "Queue1" + SourceAccessConfigurations: + - Type: BASIC_AUTH + URI: arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c + SecretsManagerKmsKeyId: 1abc23d4-567f-8ab9-cde0-1fab234c5d67 + FilterCriteria: + Filters: + - Pattern: '{"name": "value"}' KinesisStream: Type: AWS::Kinesis::Stream diff --git a/tests/translator/output/aws-cn/function_with_event_filtering.json b/tests/translator/output/aws-cn/function_with_event_filtering.json index 60f4c7082..a32d89958 100644 --- a/tests/translator/output/aws-cn/function_with_event_filtering.json +++ b/tests/translator/output/aws-cn/function_with_event_filtering.json @@ -58,8 +58,66 @@ "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole", "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole", + "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaMSKExecutionRole", "arn:aws-cn:iam::aws:policy/service-role/AWSLambdaSQSQueueExecutionRole" ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeVpcs", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "SelfManagedKafkaExecutionRolePolicy" + }, + { + "PolicyName": "SamAutoGeneratedAMQPolicy", + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "mq:DescribeBroker" + ], + "Effect": "Allow", + "Resource": "arn:aws:mq:us-east-2:123456789012:broker:MyBroker:b-1234a5b6-78cd-901e-2fgh-3i45j6k178l9" + }, + { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Resource": { + "Fn::Sub": "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/1abc23d4-567f-8ab9-cde0-1fab234c5d67" + } + } + ] + } + } + ], "Tags": [ { "Key": "lambda:createdBy", @@ -115,7 +173,7 @@ } } }, - "FilteredEventsFunctionMySqsQueue": { + "FilteredEventsFunctionMySqsEvent": { "Type": "AWS::Lambda::EventSourceMapping", "Properties": { "EventSourceArn": { @@ -135,6 +193,90 @@ ] } } + }, + "FilteredEventsFunctionMSKEvent": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": "arn:aws:kafka:us-west-2:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2", + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "StartingPosition": "LATEST", + "Topics": [ + "MyDummyTestTopic" + ], + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } + }, + "FilteredEventsFunctionMyKafkaEvent": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "Topics": [ + "Topic1" + ], + "SourceAccessConfigurations": [ + { + "Type": "SASL_SCRAM_512_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Type": "VPC_SUBNET", + "URI": "subnet:subnet-12345" + }, + { + "Type": "VPC_SECURITY_GROUP", + "URI": "security_group:sg-67890" + } + ], + "SelfManagedEventSource": { + "Endpoints": { + "KafkaBootstrapServers": [ + "abc.xyz.com:9092" + ] + } + }, + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } + }, + "FilteredEventsFunctionMyMQQueue": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": "arn:aws:mq:us-east-2:123456789012:broker:MyBroker:b-1234a5b6-78cd-901e-2fgh-3i45j6k178l9", + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "Queues": [ + "Queue1" + ], + "SourceAccessConfigurations": [ + { + "Type": "BASIC_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + } + ], + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } } } } \ No newline at end of file diff --git a/tests/translator/output/aws-us-gov/function_with_event_filtering.json b/tests/translator/output/aws-us-gov/function_with_event_filtering.json index ca2fc7d17..6411fd3c7 100644 --- a/tests/translator/output/aws-us-gov/function_with_event_filtering.json +++ b/tests/translator/output/aws-us-gov/function_with_event_filtering.json @@ -58,8 +58,66 @@ "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole", "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole", + "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaMSKExecutionRole", "arn:aws-us-gov:iam::aws:policy/service-role/AWSLambdaSQSQueueExecutionRole" ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeVpcs", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "SelfManagedKafkaExecutionRolePolicy" + }, + { + "PolicyName": "SamAutoGeneratedAMQPolicy", + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "mq:DescribeBroker" + ], + "Effect": "Allow", + "Resource": "arn:aws:mq:us-east-2:123456789012:broker:MyBroker:b-1234a5b6-78cd-901e-2fgh-3i45j6k178l9" + }, + { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Resource": { + "Fn::Sub": "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/1abc23d4-567f-8ab9-cde0-1fab234c5d67" + } + } + ] + } + } + ], "Tags": [ { "Key": "lambda:createdBy", @@ -115,7 +173,7 @@ } } }, - "FilteredEventsFunctionMySqsQueue": { + "FilteredEventsFunctionMySqsEvent": { "Type": "AWS::Lambda::EventSourceMapping", "Properties": { "EventSourceArn": { @@ -135,6 +193,90 @@ ] } } + }, + "FilteredEventsFunctionMSKEvent": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": "arn:aws:kafka:us-west-2:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2", + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "StartingPosition": "LATEST", + "Topics": [ + "MyDummyTestTopic" + ], + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } + }, + "FilteredEventsFunctionMyKafkaEvent": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "Topics": [ + "Topic1" + ], + "SourceAccessConfigurations": [ + { + "Type": "SASL_SCRAM_512_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Type": "VPC_SUBNET", + "URI": "subnet:subnet-12345" + }, + { + "Type": "VPC_SECURITY_GROUP", + "URI": "security_group:sg-67890" + } + ], + "SelfManagedEventSource": { + "Endpoints": { + "KafkaBootstrapServers": [ + "abc.xyz.com:9092" + ] + } + }, + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } + }, + "FilteredEventsFunctionMyMQQueue": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": "arn:aws:mq:us-east-2:123456789012:broker:MyBroker:b-1234a5b6-78cd-901e-2fgh-3i45j6k178l9", + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "Queues": [ + "Queue1" + ], + "SourceAccessConfigurations": [ + { + "Type": "BASIC_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + } + ], + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } } } } \ No newline at end of file diff --git a/tests/translator/output/error_event_filtering.json b/tests/translator/output/error_event_filtering.json index 86e8ecf15..fb38fcf76 100644 --- a/tests/translator/output/error_event_filtering.json +++ b/tests/translator/output/error_event_filtering.json @@ -1,3 +1,3 @@ { - "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 3. Resource with id [NotSupportedPullEvent] is invalid. Event with id [KafkaEvent] is invalid. FilterCriteria is only available for DynamoDB, Kinesis, SQS events. Resource with id [NotSupportedPushEventSNSEvent] is invalid. property FilterCriteria not defined for resource of type SNS Resource with id [WrongFilterName] is invalid. Event with id [DynamoDBStreamEvent] is invalid. FilterCriteria field has a wrong format" + "errorMessage": "Invalid Serverless Application Specification document. Number of errors found: 2. Resource with id [NotSupportedPushEventSNSEvent] is invalid. property FilterCriteria not defined for resource of type SNS Resource with id [WrongFilterName] is invalid. Event with id [DynamoDBStreamEvent] is invalid. FilterCriteria field has a wrong format" } \ No newline at end of file diff --git a/tests/translator/output/function_with_event_filtering.json b/tests/translator/output/function_with_event_filtering.json index 60edd1607..688dbd489 100644 --- a/tests/translator/output/function_with_event_filtering.json +++ b/tests/translator/output/function_with_event_filtering.json @@ -58,8 +58,66 @@ "arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", "arn:aws:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole", "arn:aws:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole", + "arn:aws:iam::aws:policy/service-role/AWSLambdaMSKExecutionRole", "arn:aws:iam::aws:policy/service-role/AWSLambdaSQSQueueExecutionRole" ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "ec2:CreateNetworkInterface", + "ec2:DescribeNetworkInterfaces", + "ec2:DeleteNetworkInterface", + "ec2:DescribeVpcs", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups" + ], + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "SelfManagedKafkaExecutionRolePolicy" + }, + { + "PolicyName": "SamAutoGeneratedAMQPolicy", + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "secretsmanager:GetSecretValue" + ], + "Effect": "Allow", + "Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Action": [ + "mq:DescribeBroker" + ], + "Effect": "Allow", + "Resource": "arn:aws:mq:us-east-2:123456789012:broker:MyBroker:b-1234a5b6-78cd-901e-2fgh-3i45j6k178l9" + }, + { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Resource": { + "Fn::Sub": "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/1abc23d4-567f-8ab9-cde0-1fab234c5d67" + } + } + ] + } + } + ], "Tags": [ { "Key": "lambda:createdBy", @@ -115,7 +173,7 @@ } } }, - "FilteredEventsFunctionMySqsQueue": { + "FilteredEventsFunctionMySqsEvent": { "Type": "AWS::Lambda::EventSourceMapping", "Properties": { "EventSourceArn": { @@ -135,6 +193,90 @@ ] } } + }, + "FilteredEventsFunctionMSKEvent": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": "arn:aws:kafka:us-west-2:012345678901:cluster/mycluster/6cc0432b-8618-4f44-bccc-e1fbd8fb7c4d-2", + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "StartingPosition": "LATEST", + "Topics": [ + "MyDummyTestTopic" + ], + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } + }, + "FilteredEventsFunctionMyKafkaEvent": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "Topics": [ + "Topic1" + ], + "SourceAccessConfigurations": [ + { + "Type": "SASL_SCRAM_512_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + }, + { + "Type": "VPC_SUBNET", + "URI": "subnet:subnet-12345" + }, + { + "Type": "VPC_SECURITY_GROUP", + "URI": "security_group:sg-67890" + } + ], + "SelfManagedEventSource": { + "Endpoints": { + "KafkaBootstrapServers": [ + "abc.xyz.com:9092" + ] + } + }, + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } + }, + "FilteredEventsFunctionMyMQQueue": { + "Type": "AWS::Lambda::EventSourceMapping", + "Properties": { + "EventSourceArn": "arn:aws:mq:us-east-2:123456789012:broker:MyBroker:b-1234a5b6-78cd-901e-2fgh-3i45j6k178l9", + "FunctionName": { + "Ref": "FilteredEventsFunction" + }, + "Queues": [ + "Queue1" + ], + "SourceAccessConfigurations": [ + { + "Type": "BASIC_AUTH", + "URI": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-path/my-secret-name-1a2b3c" + } + ], + "FilterCriteria": { + "Filters": [ + { + "Pattern": "{\"name\": \"value\"}" + } + ] + } + } } } } \ No newline at end of file From b85d7182866079fbb6c86f2b804e4e90a0463ba0 Mon Sep 17 00:00:00 2001 From: hnnasit <84355507+hnnasit@users.noreply.github.com> Date: Mon, 3 Oct 2022 16:05:02 -0700 Subject: [PATCH 04/11] Fixed indentation in custom_domains_edge yaml file (#273) --- .../templates/combination/api_with_custom_domains_edge.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/resources/templates/combination/api_with_custom_domains_edge.yaml b/integration/resources/templates/combination/api_with_custom_domains_edge.yaml index 582468f84..8072fc2da 100644 --- a/integration/resources/templates/combination/api_with_custom_domains_edge.yaml +++ b/integration/resources/templates/combination/api_with_custom_domains_edge.yaml @@ -43,6 +43,6 @@ Resources: BasePath: - /get Route53: - HostedZoneId: - Ref: HostedZoneId + HostedZoneId: + Ref: HostedZoneId IpV6: true \ No newline at end of file From 8490c0897554b0794a25a9b6bd655fc22f5e13b3 Mon Sep 17 00:00:00 2001 From: Chris Rehn <1280602+hoffa@users.noreply.github.com> Date: Mon, 3 Oct 2022 21:29:12 -0700 Subject: [PATCH 05/11] Always copy profile (#274) --- samtranslator/model/connector_profiles/profile.py | 5 ++++- tests/model/connector_profiles/test_profile.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/samtranslator/model/connector_profiles/profile.py b/samtranslator/model/connector_profiles/profile.py index 556fd61c1..491d1bb65 100644 --- a/samtranslator/model/connector_profiles/profile.py +++ b/samtranslator/model/connector_profiles/profile.py @@ -1,3 +1,4 @@ +import copy import json import os import re @@ -10,7 +11,9 @@ def get_profile(source_type: str, dest_type: str): - return PROFILE["Permissions"].get(source_type, {}).get(dest_type) + profile = PROFILE["Permissions"].get(source_type, {}).get(dest_type) + # Ensure not passing a mutable shared variable + return copy.deepcopy(profile) def verify_profile_variables_replaced(obj: Any) -> None: diff --git a/tests/model/connector_profiles/test_profile.py b/tests/model/connector_profiles/test_profile.py index e621e2c52..bcde3cca5 100644 --- a/tests/model/connector_profiles/test_profile.py +++ b/tests/model/connector_profiles/test_profile.py @@ -2,7 +2,11 @@ from parameterized import parameterized -from samtranslator.model.connector_profiles.profile import profile_replace, verify_profile_variables_replaced +from samtranslator.model.connector_profiles.profile import ( + get_profile, + profile_replace, + verify_profile_variables_replaced, +) class TestProfile(TestCase): @@ -235,3 +239,9 @@ def test_verify_not_replaced(self, profile, error_includes): with self.assertRaises(ValueError) as ctx: verify_profile_variables_replaced(profile) self.assertIn(error_includes, str(ctx.exception)) + + def test_get_profile_copied(self): + d1 = get_profile("AWS::Lambda::Function", "AWS::DynamoDB::Table") + d1["Type"] = "overridden" + d2 = get_profile("AWS::Lambda::Function", "AWS::DynamoDB::Table") + self.assertNotEqual(d1, d2) From f491eb264c54fd2d7cbdeae658065b3c2c09c678 Mon Sep 17 00:00:00 2001 From: Chris Rehn <1280602+hoffa@users.noreply.github.com> Date: Tue, 4 Oct 2022 09:33:14 -0700 Subject: [PATCH 06/11] Increase api_with_authorizer retry delay (#275) --- integration/combination/test_api_with_authorizer_apikey.py | 2 +- integration/combination/test_api_with_authorizers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/combination/test_api_with_authorizer_apikey.py b/integration/combination/test_api_with_authorizer_apikey.py index 8f8f04aa1..cdbb2e4e0 100644 --- a/integration/combination/test_api_with_authorizer_apikey.py +++ b/integration/combination/test_api_with_authorizer_apikey.py @@ -71,7 +71,7 @@ def test_authorizer_apikey(self): # ApiKeySourceType is AUTHORIZER. Passing api key via x-api-key will not get authorized self.verify_authorized_request(base_url + "lambda-token-api-key", 401, "x-api-key", key["value"]) - @retry(StatusCodeError, 10) + @retry(StatusCodeError, 10, 0.25) def verify_authorized_request( self, url, diff --git a/integration/combination/test_api_with_authorizers.py b/integration/combination/test_api_with_authorizers.py index 26780c0a7..b0188a359 100644 --- a/integration/combination/test_api_with_authorizers.py +++ b/integration/combination/test_api_with_authorizers.py @@ -427,7 +427,7 @@ def test_authorizers_with_invoke_function_set_none(self): auth_type_for_api_event_without_auth = api_event_with_out_auth["authorizationType"] self.assertEqual(auth_type_for_api_event_without_auth, "NONE") - @retry(StatusCodeError, 10) + @retry(StatusCodeError, 10, 0.25) def verify_authorized_request( self, url, From 6a65cb24ec74a273bc7291737c279fc795036465 Mon Sep 17 00:00:00 2001 From: Chris Rehn <1280602+hoffa@users.noreply.github.com> Date: Tue, 4 Oct 2022 09:51:37 -0700 Subject: [PATCH 07/11] Add retry-once to connector tests (#276) --- integration/combination/test_connectors.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/integration/combination/test_connectors.py b/integration/combination/test_connectors.py index 6b64fc2d5..09c6cc9cd 100644 --- a/integration/combination/test_connectors.py +++ b/integration/combination/test_connectors.py @@ -1,9 +1,12 @@ from time import sleep from parameterized import parameterized +from tenacity import retry, stop_after_attempt from integration.conftest import clean_bucket from integration.helpers.base_test import S3_BUCKET_PREFIX, BaseTest from integration.helpers.resource import generate_suffix +retry_once = retry(stop=stop_after_attempt(2)) + class TestConnectors(BaseTest): def tearDown(self): @@ -41,6 +44,7 @@ def tearDown(self): ("combination/connector_table_to_function_read",), ] ) + @retry_once def test_connector_by_invoking_a_function(self, template_file_path): self.skip_using_service_detector(template_file_path) self.create_and_verify_stack(template_file_path) @@ -72,6 +76,7 @@ def test_connector_by_invoking_a_function(self, template_file_path): ("combination/connector_sfn_to_eb_custom_write",), ] ) + @retry_once def test_connector_by_sync_execute_an_state_machine(self, template_file_path): self.skip_using_service_detector(template_file_path) self.create_and_verify_stack(template_file_path) @@ -91,6 +96,7 @@ def test_connector_by_sync_execute_an_state_machine(self, template_file_path): ("combination/connector_sfn_to_sfn_sync",), ] ) + @retry_once def test_connector_by_async_execute_an_state_machine(self, template_file_path): self.skip_using_service_detector(template_file_path) self.create_and_verify_stack(template_file_path) @@ -123,6 +129,7 @@ def test_connector_by_async_execute_an_state_machine(self, template_file_path): ("combination/connector_bucket_to_function_write",), ] ) + @retry_once def test_connector_by_execute_a_s3_bucket(self, template_file_path): self.skip_using_service_detector(template_file_path) bucket_name = S3_BUCKET_PREFIX + "connector" + generate_suffix() From 037d17a7b818ce9e01293d2e3c10b11408693625 Mon Sep 17 00:00:00 2001 From: _sam <3804518+aahung@users.noreply.github.com> Date: Tue, 4 Oct 2022 10:44:37 -0700 Subject: [PATCH 08/11] Use node14 runtime in connector integ tests (#278) --- .../combination/connector_event_rule_to_eb_custom_write.yaml | 2 +- .../combination/connector_event_rule_to_eb_default_write.yaml | 2 +- .../combination/connector_event_rule_to_lambda_write.yaml | 4 ++-- .../combination/connector_event_rule_to_sfn_write.yaml | 2 +- .../combination/connector_event_rule_to_sns_write.yaml | 2 +- .../combination/connector_event_rule_to_sqs_write.yaml | 2 +- .../templates/combination/connector_httpapi_to_function.yaml | 2 +- .../templates/combination/connector_restapi_to_function.yaml | 2 +- .../combination/connector_sfn_to_function_write.yaml | 2 +- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/integration/resources/templates/combination/connector_event_rule_to_eb_custom_write.yaml b/integration/resources/templates/combination/connector_event_rule_to_eb_custom_write.yaml index b9d44e2cd..f5c28fa4a 100644 --- a/integration/resources/templates/combination/connector_event_rule_to_eb_custom_write.yaml +++ b/integration/resources/templates/combination/connector_event_rule_to_eb_custom_write.yaml @@ -2,7 +2,7 @@ Resources: TriggerFunction: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Timeout: 10 # in case eb has delay InlineCode: | diff --git a/integration/resources/templates/combination/connector_event_rule_to_eb_default_write.yaml b/integration/resources/templates/combination/connector_event_rule_to_eb_default_write.yaml index 726fcb389..6c7afd27e 100644 --- a/integration/resources/templates/combination/connector_event_rule_to_eb_default_write.yaml +++ b/integration/resources/templates/combination/connector_event_rule_to_eb_default_write.yaml @@ -2,7 +2,7 @@ Resources: TriggerFunction: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Timeout: 10 # in case eb has delay InlineCode: | diff --git a/integration/resources/templates/combination/connector_event_rule_to_lambda_write.yaml b/integration/resources/templates/combination/connector_event_rule_to_lambda_write.yaml index 18c7b8e82..9e1740731 100644 --- a/integration/resources/templates/combination/connector_event_rule_to_lambda_write.yaml +++ b/integration/resources/templates/combination/connector_event_rule_to_lambda_write.yaml @@ -2,7 +2,7 @@ Resources: TriggerFunction: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Timeout: 10 # in case eb has delay InlineCode: | @@ -52,7 +52,7 @@ Resources: Function: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler InlineCode: | const AWS = require('aws-sdk'); diff --git a/integration/resources/templates/combination/connector_event_rule_to_sfn_write.yaml b/integration/resources/templates/combination/connector_event_rule_to_sfn_write.yaml index 688037cd9..4c2306fa9 100644 --- a/integration/resources/templates/combination/connector_event_rule_to_sfn_write.yaml +++ b/integration/resources/templates/combination/connector_event_rule_to_sfn_write.yaml @@ -2,7 +2,7 @@ Resources: TriggerFunction: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Timeout: 10 # in case eb has delay InlineCode: | diff --git a/integration/resources/templates/combination/connector_event_rule_to_sns_write.yaml b/integration/resources/templates/combination/connector_event_rule_to_sns_write.yaml index f0b265574..12fd5f6bf 100644 --- a/integration/resources/templates/combination/connector_event_rule_to_sns_write.yaml +++ b/integration/resources/templates/combination/connector_event_rule_to_sns_write.yaml @@ -2,7 +2,7 @@ Resources: TriggerFunction: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Timeout: 10 # in case eb has delay InlineCode: | diff --git a/integration/resources/templates/combination/connector_event_rule_to_sqs_write.yaml b/integration/resources/templates/combination/connector_event_rule_to_sqs_write.yaml index d4b42a031..a1420a2a3 100644 --- a/integration/resources/templates/combination/connector_event_rule_to_sqs_write.yaml +++ b/integration/resources/templates/combination/connector_event_rule_to_sqs_write.yaml @@ -2,7 +2,7 @@ Resources: TriggerFunction: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Timeout: 10 # in case eb has delay InlineCode: | diff --git a/integration/resources/templates/combination/connector_httpapi_to_function.yaml b/integration/resources/templates/combination/connector_httpapi_to_function.yaml index c94f08b21..813410733 100644 --- a/integration/resources/templates/combination/connector_httpapi_to_function.yaml +++ b/integration/resources/templates/combination/connector_httpapi_to_function.yaml @@ -27,7 +27,7 @@ Resources: Type: AWS::Lambda::Function Properties: Role: !GetAtt MyRole1.Arn - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Code: ZipFile: | diff --git a/integration/resources/templates/combination/connector_restapi_to_function.yaml b/integration/resources/templates/combination/connector_restapi_to_function.yaml index 9710250e1..10a5da1aa 100644 --- a/integration/resources/templates/combination/connector_restapi_to_function.yaml +++ b/integration/resources/templates/combination/connector_restapi_to_function.yaml @@ -27,7 +27,7 @@ Resources: Type: AWS::Lambda::Function Properties: Role: !GetAtt MyRole1.Arn - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler Code: ZipFile: | diff --git a/integration/resources/templates/combination/connector_sfn_to_function_write.yaml b/integration/resources/templates/combination/connector_sfn_to_function_write.yaml index ce7ee633d..8cfbbcbf0 100644 --- a/integration/resources/templates/combination/connector_sfn_to_function_write.yaml +++ b/integration/resources/templates/combination/connector_sfn_to_function_write.yaml @@ -19,7 +19,7 @@ Resources: MyFunction: Type: AWS::Serverless::Function Properties: - Runtime: nodejs16.x + Runtime: nodejs14.x Handler: index.handler InlineCode: | exports.handler = async (event) => { From 97a82b953c7cdd4f05b5c6fa3cad06be19e3c5b1 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Tue, 4 Oct 2022 12:35:35 -0700 Subject: [PATCH 09/11] add a step to clean up unused network interfaces in the precreated subnets (#277) --- integration/conftest.py | 30 ++++++++++++++++++++++++++ integration/helpers/client_provider.py | 11 ++++++++++ 2 files changed, 41 insertions(+) diff --git a/integration/conftest.py b/integration/conftest.py index babdd4ecd..c78661f78 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -1,3 +1,4 @@ +import time import boto3 import botocore import pytest @@ -63,6 +64,26 @@ def clean_all_integ_buckets(): clean_bucket(bucket.name, s3_client) +def _delete_unused_network_interface_by_subnet(ec2_client, subnet_id): + """Deletes unused network interface under the provided subnet""" + paginator = ec2_client.get_paginator("describe_network_interfaces") + response_iterator = paginator.paginate( + Filters=[ + {"Name": "subnet-id", "Values": [subnet_id]}, + {"Name": "status", "Values": ["available"]}, + ] + ) + network_interface_ids = [] + for page in response_iterator: + network_interface_ids += [ni["NetworkInterfaceId"] for ni in page["NetworkInterfaces"]] + + for ni_id in network_interface_ids: + ec2_client.delete_network_interface(NetworkInterfaceId=ni_id) + time.sleep(0.5) + + LOG.info("Deleted %s unused network interfaces under subnet %s", len(network_interface_ids), subnet_id) + + @pytest.fixture() def setup_companion_stack_once(tmpdir_factory, get_prefix): tests_integ_dir = Path(__file__).resolve().parents[1] @@ -74,6 +95,15 @@ def setup_companion_stack_once(tmpdir_factory, get_prefix): companion_stack = Stack(stack_name, companion_stack_tempalte_path, cfn_client, output_dir) companion_stack.create_or_update(_stack_exists(stack_name)) + ec2_client = ClientProvider().ec2_client + precreated_subnet_ids = [ + resource["PhysicalResourceId"] + for resource in companion_stack.stack_resources["StackResourceSummaries"] + if resource["LogicalResourceId"].startswith("PreCreatedSubnet") + ] + for subnet_id in precreated_subnet_ids: + _delete_unused_network_interface_by_subnet(ec2_client, subnet_id) + @pytest.fixture() def get_serverless_application_repository_app(): diff --git a/integration/helpers/client_provider.py b/integration/helpers/client_provider.py index d5958435d..cee437823 100644 --- a/integration/helpers/client_provider.py +++ b/integration/helpers/client_provider.py @@ -25,6 +25,7 @@ def __init__(self): self._kafka_client = None self._code_deploy_client = None self._sar_client = None + self._ec2_client = None @property def cfn_client(self): @@ -216,3 +217,13 @@ def sar_client(self): if not self._sar_client: self._sar_client = boto3.client("serverlessrepo") return self._sar_client + + @property + def ec2_client(self): + """ + EC2 Client + """ + with self._lock: + if not self._ec2_client: + self._ec2_client = boto3.client("ec2") + return self._ec2_client From 28cf0952e2048be2918fe0588dbbe71cf32ca0d3 Mon Sep 17 00:00:00 2001 From: Chris Rehn <1280602+hoffa@users.noreply.github.com> Date: Tue, 4 Oct 2022 12:58:37 -0700 Subject: [PATCH 10/11] Do not retry on SkipTest (#280) --- integration/combination/test_connectors.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/integration/combination/test_connectors.py b/integration/combination/test_connectors.py index 09c6cc9cd..1fa84123c 100644 --- a/integration/combination/test_connectors.py +++ b/integration/combination/test_connectors.py @@ -1,11 +1,16 @@ from time import sleep +from unittest import SkipTest from parameterized import parameterized -from tenacity import retry, stop_after_attempt +from tenacity import retry, stop_after_attempt, retry_if_exception from integration.conftest import clean_bucket from integration.helpers.base_test import S3_BUCKET_PREFIX, BaseTest from integration.helpers.resource import generate_suffix -retry_once = retry(stop=stop_after_attempt(2)) +retry_once = retry( + stop=stop_after_attempt(2), + # unittest raises SkipTest for skipping tests + retry=retry_if_exception(lambda e: not isinstance(e, SkipTest)), +) class TestConnectors(BaseTest): From ee3f28f8904100da118185a0da625c267530e223 Mon Sep 17 00:00:00 2001 From: aws-sam-cli-bot <46753707+aws-sam-cli-bot@users.noreply.github.com> Date: Tue, 11 Oct 2022 10:10:52 -0700 Subject: [PATCH 11/11] chore: bump version to 1.53.0 --- samtranslator/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samtranslator/__init__.py b/samtranslator/__init__.py index 30b35ef38..e2288504b 100644 --- a/samtranslator/__init__.py +++ b/samtranslator/__init__.py @@ -1 +1 @@ -__version__ = "1.51.0" +__version__ = "1.53.0"