test: run by canary (#1793)

* Disable test skipping if triggered by Canary.

* Adding comments for the change.

* Using False instead of None.

tests/integration/deploy/test_deploy_command.py

@@ -12,11 +12,11 @@
 from samcli.lib.bootstrap.bootstrap import SAM_CLI_STACK_NAME
 from tests.integration.deploy.deploy_integ_base import DeployIntegBase
 from tests.integration.package.package_integ_base import PackageIntegBase
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
 
 # Deploy tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
-# This is to restrict package tests to run outside of CI/CD and when the branch is not master.
-SKIP_DEPLOY_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI
+# This is to restrict package tests to run outside of CI/CD, when the branch is not master or tests are not run by Canary.
+SKIP_DEPLOY_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 CFN_SLEEP = 3
 TIMEOUT = 300
 

tests/integration/init/schemas/test_init_with_schemas_command.py

@@ -7,10 +7,10 @@
 from pathlib import Path
 
 from tests.integration.init.schemas.schemas_test_data_setup import SchemaTestDataSetup
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
 
 # Schemas tests require credentials. This is to skip running the test where credentials are not available.
-SKIP_SCHEMA_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI
+SKIP_SCHEMA_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 
 
 @skipIf(SKIP_SCHEMA_TESTS, "Skip schema test")

tests/integration/local/invoke/test_integrations_cli.py

@@ -13,11 +13,11 @@
 
 from tests.integration.local.invoke.layer_utils import LayerUtils
 from .invoke_integ_base import InvokeIntegBase
-from tests.testing_utils import IS_WINDOWS, RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI
+from tests.testing_utils import IS_WINDOWS, RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
 
 # Layers tests require credentials and Travis will only add credentials to the env if the PR is from the same repo.
-# This is to restrict layers tests to run outside of Travis and when the branch is not master.
-SKIP_LAYERS_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI
+# This is to restrict layers tests to run outside of Travis, when the branch is not master and tests are not run by Canary.
+SKIP_LAYERS_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 
 from pathlib import Path
 

tests/integration/package/test_package_command.py

@@ -5,11 +5,11 @@
 from parameterized import parameterized
 
 from .package_integ_base import PackageIntegBase
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
 
 # Package tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
-# This is to restrict package tests to run outside of CI/CD and when the branch is not master.
-SKIP_PACKAGE_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI
+# This is to restrict package tests to run outside of CI/CD, when the branch is not master and tests are not run by Canary.
+SKIP_PACKAGE_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 TIMEOUT = 300
 
 

tests/integration/publish/test_command_integ.py

@@ -7,11 +7,11 @@
 
 from samcli.commands.publish.command import SEMANTIC_VERSION
 from .publish_app_integ_base import PublishAppIntegBase
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
 
 # Publish tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
-# This is to restrict publish tests to run outside of CI/CD and when the branch is not master.
-SKIP_PUBLISH_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI
+# This is to restrict publish tests to run outside of CI/CD, when the branch is not master and tests are not run by Canary.
+SKIP_PUBLISH_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 TIMEOUT = 300
 
 

tests/regression/deploy/test_deploy_regression.py

@@ -10,11 +10,11 @@
 
 from tests.regression.deploy.regression_deploy_base import DeployRegressionBase
 from tests.regression.package.regression_package_base import PackageRegressionBase
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
 
 # Package Regression tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
-# This is to restrict package tests to run outside of CI/CD and when the branch is not master.
-SKIP_DEPLOY_REGRESSION_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI
+# This is to restrict package tests to run outside of CI/CD, when the branch is not master and tests are not run by Canary.
+SKIP_DEPLOY_REGRESSION_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 CFN_SLEEP = 3
 TIMEOUT = 300
 # Only testing return codes to be equivalent

tests/regression/package/test_package_regression.py

@@ -2,11 +2,11 @@
 from parameterized import parameterized
 
 from .regression_package_base import PackageRegressionBase
-from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI
+from tests.testing_utils import RUNNING_ON_CI, RUNNING_TEST_FOR_MASTER_ON_CI, RUN_BY_CANARY
 
 # Package Regression tests require credentials and CI/CD will only add credentials to the env if the PR is from the same repo.
-# This is to restrict package tests to run outside of CI/CD and when the branch is not master.
-SKIP_PACKAGE_REGRESSION_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI
+# This is to restrict package tests to run outside of CI/CD, when the branch is not master and tests are not run by Canary.
+SKIP_PACKAGE_REGRESSION_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI and not RUN_BY_CANARY
 
 
 # Only tested cases where the output template file changes, adding metadata or kms keys does not change the output.

tests/testing_utils.py

@@ -8,6 +8,7 @@
 RUNNING_ON_CI = os.environ.get("APPVEYOR", False)
 RUNNING_TEST_FOR_MASTER_ON_CI = os.environ.get("APPVEYOR_REPO_BRANCH", "master") != "master"
 CI_OVERRIDE = os.environ.get("APPVEYOR_CI_OVERRIDE", False)
+RUN_BY_CANARY = os.environ.get("BY_CANARY", False)
 
 
 class FileCreator(object):