feat: adding separate bucket based on python version for integration …

…tests. (#1828)

* feat: adding separate bucket based on python version for integration tests.

* Better code formatting.

* Code cleanup after udpating AppVeyor.

appveyor.yml

@@ -15,6 +15,7 @@ environment:
       NOSE_PARAMETERIZED_NO_WARN: 1
       INSTALL_PY_37_PIP: 1
       INSTALL_PY_38_PIP: 1
+      AWS_S3: 'AWS_S3_36'
 
     - PYTHON_HOME: "C:\\Python37-x64"
       PYTHON_VERSION: '3.7.4'
@@ -23,6 +24,7 @@ environment:
       NOSE_PARAMETERIZED_NO_WARN: 1
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_38_PIP: 1
+      AWS_S3: 'AWS_S3_37'
 
     - PYTHON_HOME: "C:\\Python38-x64"
       PYTHON_VERSION: '3.8.0'
@@ -31,6 +33,7 @@ environment:
       NOSE_PARAMETERIZED_NO_WARN: 1
       INSTALL_PY_36_PIP: 1
       INSTALL_PY_37_PIP: 1
+      AWS_S3: 'AWS_S3_38'
 
 for:
   - 

tests/integration/package/package_integ_base.py

@@ -14,7 +14,22 @@ class PackageIntegBase(TestCase):
     @classmethod
     def setUpClass(cls):
         cls.region_name = os.environ.get("AWS_DEFAULT_REGION")
-        cls.pre_created_bucket = os.environ.get("AWS_S3", False)
+        """
+        Our integration tests use S3 bucket to run several tests. Given that S3 objects are eventually consistent
+        and we are using same bucket for lot of integration tests, we want to have multiple buckets to reduce
+        transient failures. In order to achieve this we created 3 buckets one for each python version we support (3.6,
+        3.7 and 3.8). Tests running for respective python version will use respective bucket.
+
+        AWS_S3 will point to a new environment variable AWS_S3_36 or AWS_S3_37 or AWS_S3_38. This is controlled by
+        Appveyor. These environment variables will hold bucket name to run integration tests. Eg:
+
+        For Python36:
+        AWS_S3=AWS_S3_36
+        AWS_S3_36=aws-sam-cli-canary-region-awssamclitestbucket-forpython36
+
+        For backwards compatibility we are falling back to reading AWS_S3 so that current tests keep working.
+        """
+        cls.pre_created_bucket = os.environ.get(os.environ.get("AWS_S3"), False)
         cls.bucket_name = cls.pre_created_bucket if cls.pre_created_bucket else str(uuid.uuid4())
         cls.test_data_path = Path(__file__).resolve().parents[1].joinpath("testdata", "package")
 

tests/integration/publish/publish_app_integ_base.py

@@ -16,7 +16,8 @@ class PublishAppIntegBase(TestCase):
     @classmethod
     def setUpClass(cls):
         cls.region_name = os.environ.get("AWS_DEFAULT_REGION")
-        cls.pre_created_bucket = os.environ.get("AWS_S3", False)
+        """Please read comments in package_integ_base.py for more details around this."""
+        cls.pre_created_bucket = os.environ.get(os.environ.get("AWS_S3"), False)
         cls.bucket_name = cls.pre_created_bucket if cls.pre_created_bucket else str(uuid.uuid4())
         cls.bucket_name_placeholder = "<bucket-name>"
         cls.application_name_placeholder = "<application-name>"

tests/regression/package/regression_package_base.py

@@ -17,7 +17,8 @@ class PackageRegressionBase(TestCase):
     @classmethod
     def setUpClass(cls):
         cls.region_name = os.environ.get("AWS_DEFAULT_REGION")
-        cls.pre_created_bucket = os.environ.get("AWS_S3", False)
+        """Please read comments in package_integ_base.py for more details around this."""
+        cls.pre_created_bucket = os.environ.get(os.environ.get("AWS_S3"), False)
         cls.bucket_name = cls.pre_created_bucket if cls.pre_created_bucket else str(uuid.uuid4())
         cls.test_data_path = Path(__file__).resolve().parents[2].joinpath("integration", "testdata", "package")
 

tests/testing_utils.py

@@ -3,7 +3,6 @@
 import tempfile
 import shutil
 
-
 IS_WINDOWS = platform.system().lower() == "windows"
 RUNNING_ON_CI = os.environ.get("APPVEYOR", False)
 RUNNING_TEST_FOR_MASTER_ON_CI = os.environ.get("APPVEYOR_REPO_BRANCH", "master") != "master"