chore: sync foss changes (#2445)

* chore: sync foss changes

* Add make target for creating companion stack, add config files

* Fix cors tests

* add --no-cov to companion setup

* fix wrong test setups

* fix wrong test setups

* use parameters when doing local transform

* Revert "use parameters when doing local transform"

This reverts commit 4b987dc.

Co-authored-by: Daniel Mil <mladan@amazon.com>

INTEGRATION_TESTS.md

@@ -34,6 +34,14 @@ If you haven't done so already, run the following command in a terminal at the r
 make init
 ```
 
+### Setting up a companion stack
+
+To run the tests, a companion stack first needs to be created. This stack houses some resources that are required by the tests, such as an S3 bucket.
+
+```
+make prepare-companion-stack
+```
+
 ### Running all the tests
 
 From the root of the repository, run:

Makefile

@@ -27,6 +27,9 @@ lint:
 	# Linter performs static analysis to catch latent bugs
 	pylint --rcfile .pylintrc samtranslator
 
+prepare-companion-stack:
+	pytest -v --no-cov integration/setup -m setup
+
 # Command to run everytime you make changes to verify everything works
 dev: test
 
@@ -43,5 +46,6 @@ TARGETS
 	integ-test  Run the Integration tests.
 	dev         Run all development tests after a change.
 	pr          Perform all checks before submitting a Pull Request.
+	prepare-companion-stack    Create or update the companion stack for running integration tests.
 
 endef

integration/combination/test_custom_http_api_domains_test.py

@@ -2,7 +2,6 @@
 
 from integration.config.service_names import CUSTOM_DOMAIN
 from integration.helpers.base_internal_test import BaseInternalTest
-from integration.helpers.file_resources import FILE_TO_S3_URI_MAP
 from integration.helpers.resource import current_region_not_included
 
 
@@ -25,7 +24,7 @@ def test_custom_http_api_domains_regional(self):
         self.assertEqual("httpapi.sam-gamma-regional.com", result["DomainName"])
 
         mtls_auth_config = result["MutualTlsAuthentication"]
-        self.assertEqual(FILE_TO_S3_URI_MAP["MTLSCert.pem"]["uri"], mtls_auth_config["TruststoreUri"])
+        self.assertEqual(self.file_to_s3_uri_map["MTLSCert.pem"]["uri"], mtls_auth_config["TruststoreUri"])
 
         domain_name_configs = result["DomainNameConfigurations"]
         self.assertEqual(1, len(domain_name_configs))

integration/combination/test_custom_rest_api_domains.py

@@ -2,7 +2,6 @@
 
 from integration.config.service_names import CUSTOM_DOMAIN
 from integration.helpers.base_internal_test import BaseInternalTest
-from integration.helpers.file_resources import FILE_TO_S3_URI_MAP
 from integration.helpers.resource import current_region_not_included
 
 
@@ -47,7 +46,7 @@ def test_custom_rest_api_domains_regional(self):
         self.assertEqual("REGIONAL", end_point_types[0])
 
         mtls_auth_config = result["mutualTlsAuthentication"]
-        self.assertEqual(FILE_TO_S3_URI_MAP["MTLSCert.pem"]["uri"], mtls_auth_config["truststoreUri"])
+        self.assertEqual(self.file_to_s3_uri_map["MTLSCert.pem"]["uri"], mtls_auth_config["truststoreUri"])
 
     def test_custom_rest_api_domains_regional_ownership_verification(self):
         self.create_and_verify_stack("combination/api_with_custom_domains_regional_ownership_verification")

integration/combination/test_function_with_file_system_config.py

@@ -1,11 +1,23 @@
 from unittest.case import skipIf
+import pytest
 
 from integration.config.service_names import EFS
 from integration.helpers.base_test import BaseTest
 from integration.helpers.resource import current_region_does_not_support
 
 
 class TestFunctionWithFileSystemConfig(BaseTest):
+    @pytest.fixture(autouse=True)
+    def companion_stack_outputs(self, get_companion_stack_outputs):
+        self.companion_stack_outputs = get_companion_stack_outputs
+
     @skipIf(current_region_does_not_support([EFS]), "EFS is not supported in this testing region")
     def test_function_with_efs_integration(self):
-        self.create_and_verify_stack("combination/function_with_file_system_config")
+        parameters = self.get_parameters(self.companion_stack_outputs)
+        self.create_and_verify_stack("combination/function_with_file_system_config", parameters)
+
+    def get_parameters(self, dictionary):
+        parameters = []
+        parameters.append(self.generate_parameter("PreCreatedSubnetOne", dictionary["PreCreatedSubnetOne"]))
+        parameters.append(self.generate_parameter("PreCreatedVpc", dictionary["PreCreatedVpc"]))
+        return parameters

integration/combination/test_function_with_http_api.py

@@ -1,12 +1,18 @@
+import logging
 from unittest.case import skipIf
 
+import pytest
+
 from integration.helpers.base_test import BaseTest
 from integration.helpers.resource import current_region_does_not_support
 from integration.config.service_names import HTTP_API
 
+LOG = logging.getLogger(__name__)
+
 
 @skipIf(current_region_does_not_support([HTTP_API]), "HttpApi is not supported in this testing region")
 class TestFunctionWithHttpApi(BaseTest):
+    @pytest.mark.flaky(reruns=5)
     def test_function_with_http_api(self):
         self.create_and_verify_stack("combination/function_with_http_api")
 

integration/combination/test_intrinsic_function_support.py

@@ -1,5 +1,5 @@
-from parameterized import parameterized
 from unittest.case import skipIf
+import pytest
 
 from integration.helpers.base_test import BaseTest
 from integration.helpers.resource import current_region_does_not_support
@@ -8,18 +8,24 @@
 
 @skipIf(current_region_does_not_support([REST_API]), "Rest API is not supported in this testing region")
 class TestIntrinsicFunctionsSupport(BaseTest):
+    @pytest.fixture(autouse=True)
+    def companion_stack_outputs(self, get_companion_stack_outputs):
+        self.companion_stack_outputs = get_companion_stack_outputs
 
-    # test code definition uri object and serverless function properties support
-    @parameterized.expand(
-        [
-            "combination/intrinsics_code_definition_uri",
-            "combination/intrinsics_serverless_function",
-        ]
-    )
-    def test_common_support(self, file_name):
+    # test serverless function properties support
+    def test_serverless_function_property_support(self):
         # Just a simple deployment will validate that Code & Swagger files were accessible
         # Just a simple deployment will validate that all properties were resolved expected
-        self.create_and_verify_stack(file_name, self.get_default_test_template_parameters())
+        parameters = self.get_parameters(self.companion_stack_outputs)
+        parameters.extend(self.get_default_test_template_parameters())
+        self.create_and_verify_stack("combination/intrinsics_serverless_function", parameters)
+
+    # test code definition uri object support
+    def test_definition_uri_support(self):
+        # Just a simple deployment will validate that Code & Swagger files were accessible
+        # Just a simple deployment will validate that all properties were resolved expected
+        parameters = self.get_default_test_template_parameters()
+        self.create_and_verify_stack("combination/intrinsics_code_definition_uri", parameters)
 
     def test_severless_api_properties_support(self):
         self.create_and_verify_stack(
@@ -62,3 +68,9 @@ def test_severless_api_properties_support(self):
         self.assertEqual(tags["lambda:createdBy"], "SAM", "Expected 'SAM' tag value, but not found.")
         self.assertTrue("TagKey1" in tags)
         self.assertEqual(tags["TagKey1"], api_stage_name)
+
+    def get_parameters(self, dictionary):
+        parameters = []
+        parameters.append(self.generate_parameter("PreCreatedSubnetOne", dictionary["PreCreatedSubnetOne"]))
+        parameters.append(self.generate_parameter("PreCreatedVpc", dictionary["PreCreatedVpc"]))
+        return parameters

integration/config/code_key_to_file_map.json

@@ -0,0 +1,8 @@
+{
+    "codeuri": "code.zip",
+    "contenturi": "layer1.zip",
+    "definitionuri": "swagger1.json",
+    "templateurl": "template.yaml",
+    "binaryMediaCodeUri": "binary-media.zip",
+    "mtlsuri": "MTLSCert.pem"
+}

integration/config/file_to_s3_map.json

@@ -0,0 +1,11 @@
+{
+    "code.zip": {"type": "s3", "uri": ""},
+    "code2.zip": {"type": "s3", "uri": ""},
+    "layer1.zip": {"type": "s3", "uri": ""},
+    "swagger1.json": {"type": "s3", "uri": ""},
+    "swagger2.json": {"type": "s3", "uri": ""},
+    "binary-media.zip": {"type": "s3", "uri": ""},
+    "template.yaml": {"type": "http", "uri": ""},
+    "MTLSCert.pem": {"type": "s3", "uri": ""},
+    "MTLSCert-Updated.pem": {"type": "s3", "uri": ""}
+}

integration/conftest.py

@@ -6,10 +6,11 @@
 
 from integration.helpers.base_test import S3_BUCKET_PREFIX
 from integration.helpers.client_provider import ClientProvider
-from integration.helpers.deployer.exceptions.exceptions import ThrottlingError
+from integration.helpers.deployer.exceptions.exceptions import S3DoesNotExistException, ThrottlingError
 from integration.helpers.deployer.utils.retry import retry_with_exponential_backoff_and_jitter
 from integration.helpers.stack import Stack
 from integration.helpers.yaml_utils import load_yaml
+from integration.helpers.resource import read_test_config_file, write_test_config_file_to_json
 
 try:
     from pathlib import Path
@@ -63,10 +64,57 @@ def setup_companion_stack_once(tmpdir_factory, get_prefix):
     cfn_client = ClientProvider().cfn_client
     output_dir = tmpdir_factory.mktemp("data")
     stack_name = get_prefix + COMPANION_STACK_NAME
-    if _stack_exists(stack_name):
-        return
     companion_stack = Stack(stack_name, companion_stack_tempalte_path, cfn_client, output_dir)
-    companion_stack.create()
+    companion_stack.create_or_update(_stack_exists(stack_name))
+
+
+@pytest.fixture()
+def upload_resources(get_s3):
+    """
+    Creates the bucket and uploads the files used by the tests to it
+    """
+    s3_bucket = get_s3
+    if not _s3_exists(s3_bucket):
+        raise S3DoesNotExistException(get_s3, "Check companion stack status")
+    code_dir = Path(__file__).resolve().parents[0].joinpath("resources").joinpath("code")
+    file_to_s3_uri_map = read_test_config_file("file_to_s3_map.json")
+
+    if not file_to_s3_uri_map or not file_to_s3_uri_map.items():
+        LOG.debug("No resources to upload")
+        return
+
+    current_file_name = ""
+
+    try:
+        s3_client = ClientProvider().s3_client
+        session = boto3.session.Session()
+        region = session.region_name
+        for file_name, file_info in file_to_s3_uri_map.items():
+            current_file_name = file_name
+            code_path = str(Path(code_dir, file_name))
+            LOG.debug("Uploading file %s to bucket %s", file_name, s3_bucket)
+            s3_client.upload_file(code_path, s3_bucket, file_name)
+            LOG.debug("File %s uploaded successfully to bucket %s", file_name, s3_bucket)
+            file_info["uri"] = get_s3_uri(file_name, file_info["type"], s3_bucket, region)
+    except ClientError as error:
+        LOG.error("Upload of file %s to bucket %s failed", current_file_name, s3_bucket, exc_info=error)
+        raise error
+
+    write_test_config_file_to_json("file_to_s3_map_modified.json", file_to_s3_uri_map)
+
+
+def get_s3_uri(file_name, uri_type, bucket, region):
+    if uri_type == "s3":
+        return "s3://{}/{}".format(bucket, file_name)
+
+    if region == "us-east-1":
+        return "https://s3.amazonaws.com/{}/{}".format(bucket, file_name)
+    if region == "us-iso-east-1":
+        return "https://s3.us-iso-east-1.c2s.ic.gov/{}/{}".format(bucket, file_name)
+    if region == "us-isob-east-1":
+        return "https://s3.us-isob-east-1.sc2s.sgov.gov/{}/{}".format(bucket, file_name)
+
+    return "https://s3-{}.amazonaws.com/{}/{}".format(region, bucket, file_name)
 
 
 @pytest.fixture()
@@ -99,6 +147,12 @@ def get_companion_stack_outputs(get_prefix):
     return get_stack_outputs(companion_stack_description)
 
 
+@pytest.fixture()
+def get_s3(get_companion_stack_outputs):
+    s3_bucket = get_companion_stack_outputs.get("PreCreatedS3Bucket")
+    return str(s3_bucket)
+
+
 @pytest.fixture()
 def get_prefix(request):
     prefix = ""
@@ -171,3 +225,15 @@ def _stack_exists(stack_name):
         raise ex
 
     return True
+
+
+@retry_with_exponential_backoff_and_jitter(ThrottlingError, 5, 360)
+def _s3_exists(s3_bucket):
+    s3 = boto3.resource("s3")
+    bucket = s3.Bucket(s3_bucket)
+    try:
+        s3.meta.client.head_bucket(Bucket=bucket.name)
+    except ClientError:
+        return False
+
+    return True