fix: add timeouts on docker local-invoke, start-api and start-lambda integration tests

appveyor.yml

@@ -9,10 +9,6 @@ environment:
 
   matrix:
 
-    - PYTHON_HOME: "C:\\Python27-x64"
-      PYTHON_VERSION: '2.7.16'
-      PYTHON_ARCH: '32'
-
     - PYTHON_HOME: "C:\\Python36-x64"
       PYTHON_VERSION: '3.6.8'
       PYTHON_ARCH: '64'
@@ -29,18 +25,45 @@ for:
         - image: Visual Studio 2017
 
     install:
+      - "SET PATH=%PYTHON_HOME%;%PATH%"
+      - "echo %PYTHON_HOME%"
+      - "echo %PATH%"
+      - "python --version"
       # Upgrade setuptools, wheel and virtualenv
       - "python -m pip install --upgrade setuptools wheel virtualenv"
 
       # Create new virtual environment and activate it
       - "rm -rf venv"
       - "python -m virtualenv venv"
-      - "venv/Scripts/activate"
+      - "venv\\Scripts\\activate"
+
+    build_script:
+      # Activate virtualenv again on windows
+      - "venv\\Scripts\\activate"
+      - "python -c \"import sys; print(sys.executable)\""
+      - "pip install -e \".[dev]\""
+
+    test_script:
+      # Activate virtualenv again on windows
+      - "venv\\Scripts\\activate"
+      - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit"
+      - "pylint --rcfile .pylintrc samcli"
+
+      # Runs only in Linux
+      - sh: "pytest -vv tests/integration"
+      - sh: "/tmp/black --check setup.py tests samcli scripts"
+      - sh: "python scripts/check-isolated-needs-update.py"
 
+      # Smoke tests run in parallel - it runs on both Linux & Windows
+      # Presence of the RUN_SMOKE envvar will run the smoke tests
+      # Note: temporarily removing as with current dependencies we require syslog on windows
+      # which is not present on stdlib.
+      # - ps: "If ($env:RUN_SMOKE) {pytest -n 4 -vv tests/smoke}"
   - 
     matrix:
       only:
         - image: Ubuntu
+
     install:
       - sh: "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
       - sh: "PATH=$JAVA_HOME/bin:$PATH"
@@ -57,30 +80,30 @@ for:
       - sh: "wget -O /tmp/black https://github.com/python/black/releases/download/19.3b0/black"
       - sh: "chmod +x /tmp/black"
       - sh: "/tmp/black --version"
+
+    build_script:
+      - "python -c \"import sys; print(sys.executable)\""
+      - "pip install -e \".[dev]\""
+
+    test_script:
+      - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit"
+      - "pylint --rcfile .pylintrc samcli"
+
+      # Runs only in Linux
+      - sh: "pytest -vv tests/integration"
+      - sh: "/tmp/black --check setup.py tests samcli scripts"
+      - sh: "python scripts/check-isolated-needs-update.py"
+
+      # Smoke tests run in parallel - it runs on both Linux & Windows
+      # Presence of the RUN_SMOKE envvar will run the smoke tests
+      - ps: "If ($env:RUN_SMOKE) {pytest -n 4 -vv tests/smoke}"
 
   - 
     matrix:
       only:
         - ONLY_SMOKE: 1
+
     test_script:
       # Smoke tests run in parallel
-      - "pytest -n 4 -vv tests/smoke"
-
-build_script:
-  - "python -c \"import sys; print(sys.executable)\""
-
-  # Actually install SAM CLI's dependencies
-  - "pip install -e \".[dev]\""
-
-test_script:
-    - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit"
-    - "pylint --rcfile .pylintrc samcli"
-
-    # Runs only in Linux
-    - sh: "pytest -vv tests/integration"
-    - sh: "/tmp/black --check setup.py tests samcli scripts"
-    - sh: "python scripts/check-isolated-needs-update.py"
-
-    # Smoke tests run in parallel - it runs on both Linux & Windows
-    # Presence of the RUN_SMOKE envvar will run the smoke tests
-    - ps: "If ($env:RUN_SMOKE) {pytest -n 4 -vv tests/smoke}"
+      - sh: "venv/Scripts/activate"
+      - sh: "pytest -n 4 -vv tests/smoke"

requirements/dev.txt

@@ -1,13 +1,13 @@
 coverage==4.3.4
-tox==2.2.1
 pytest-cov==2.4.0
 # astroid > 2.0.4 is not compatible with pylint1.7
 astroid>=1.5.8,<2.1.0
 pylint==1.7.2
 
 # Test requirements
-pytest==3.1.0
-py==1.4.33
+pytest==3.6.0
+py==1.5.1
+pluggy==0.6.0
 mock==2.0.0
 parameterized==0.6.1
 pathlib2==2.3.2; python_version<"3.4"
@@ -16,3 +16,4 @@ futures==3.2.0; python_version<"3.2.3"
 backports.tempfile==1.0
 pytest-xdist==1.20.0
 pytest-forked==1.0.2
+pytest-timeout==1.3.3

tests/integration/local/invoke/runtimes/test_with_runtime_zips.py

@@ -5,6 +5,7 @@
 
 from subprocess import Popen, PIPE
 from nose_parameterized import parameterized, param
+import pytest
 
 from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase
 
@@ -30,6 +31,7 @@ def setUp(self):
     def tearDown(self):
         os.remove(self.events_file_path)
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     @parameterized.expand([param("Go1xFunction"), param("Java8Function")])
     def test_runtime_zip(self, function_name):
         command_list = self.get_command_list(
@@ -43,6 +45,7 @@ def test_runtime_zip(self, function_name):
         process_stdout = b"".join(process.stdout.readlines()).strip()
         self.assertEqual(process_stdout.decode("utf-8"), '"Hello World"')
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_custom_provided_runtime(self):
         command_list = self.get_command_list(
             "CustomBashFunction", template_path=self.template_path, event_path=self.events_file_path

tests/integration/local/invoke/test_integrations_cli.py

@@ -8,7 +8,7 @@
 from nose_parameterized import parameterized
 from subprocess import Popen, PIPE
 from timeit import default_timer as timer
-
+import pytest
 import docker
 
 from tests.integration.local.invoke.layer_utils import LayerUtils
@@ -28,6 +28,7 @@
 class TestSamPython36HelloWorldIntegration(InvokeIntegBase):
     template = Path("template.yml")
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_returncode_is_zero(self):
         command_list = self.get_command_list(
             "HelloWorldServerlessFunction", template_path=self.template_path, event_path=self.event_path
@@ -38,6 +39,7 @@ def test_invoke_returncode_is_zero(self):
 
         self.assertEqual(return_code, 0)
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_function_with_metadata(self):
         command_list = self.get_command_list("FunctionWithMetadata", template_path=self.template_path, no_event=True)
 
@@ -47,6 +49,7 @@ def test_function_with_metadata(self):
 
         self.assertEqual(process_stdout.decode("utf-8"), '"Hello World in a different dir"')
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_returns_execpted_results(self):
         command_list = self.get_command_list(
             "HelloWorldServerlessFunction", template_path=self.template_path, event_path=self.event_path
@@ -57,6 +60,7 @@ def test_invoke_returns_execpted_results(self):
         process_stdout = b"".join(process.stdout.readlines()).strip()
         self.assertEqual(process_stdout.decode("utf-8"), '"Hello world"')
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_of_lambda_function(self):
         command_list = self.get_command_list(
             "HelloWorldLambdaFunction", template_path=self.template_path, event_path=self.event_path
@@ -67,6 +71,7 @@ def test_invoke_of_lambda_function(self):
         process_stdout = b"".join(process.stdout.readlines()).strip()
         self.assertEqual(process_stdout.decode("utf-8"), '"Hello world"')
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     @parameterized.expand([("TimeoutFunction"), ("TimeoutFunctionWithParameter")])
     def test_invoke_with_timeout_set(self, function_name):
         command_list = self.get_command_list(
@@ -93,6 +98,7 @@ def test_invoke_with_timeout_set(self, function_name):
             msg="The return statement in the LambdaFunction " "should never return leading to an empty string",
         )
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_with_env_vars(self):
         command_list = self.get_command_list(
             "EchoCustomEnvVarFunction",
@@ -106,6 +112,7 @@ def test_invoke_with_env_vars(self):
         process_stdout = b"".join(process.stdout.readlines()).strip()
         self.assertEqual(process_stdout.decode("utf-8"), '"MyVar"')
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_when_function_writes_stdout(self):
         command_list = self.get_command_list(
             "WriteToStdoutFunction", template_path=self.template_path, event_path=self.event_path
@@ -120,6 +127,7 @@ def test_invoke_when_function_writes_stdout(self):
         self.assertIn("Docker Lambda is writing to stdout", process_stderr.decode("utf-8"))
         self.assertIn("wrote to stdout", process_stdout.decode("utf-8"))
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_when_function_writes_stderr(self):
         command_list = self.get_command_list(
             "WriteToStderrFunction", template_path=self.template_path, event_path=self.event_path
@@ -132,6 +140,7 @@ def test_invoke_when_function_writes_stderr(self):
 
         self.assertIn("Docker Lambda is writing to stderr", process_stderr.decode("utf-8"))
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_returns_expected_result_when_no_event_given(self):
         command_list = self.get_command_list("EchoEventFunction", template_path=self.template_path)
         command_list.append("--no-event")
@@ -142,6 +151,7 @@ def test_invoke_returns_expected_result_when_no_event_given(self):
         self.assertEqual(return_code, 0)
         self.assertEqual("{}", process_stdout.decode("utf-8"))
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_raises_exception_with_noargs_and_event(self):
         command_list = self.get_command_list(
             "HelloWorldLambdaFunction", template_path=self.template_path, event_path=self.event_path
@@ -154,6 +164,7 @@ def test_invoke_raises_exception_with_noargs_and_event(self):
         error_output = process_stderr.decode("utf-8")
         self.assertIn("no_event and event cannot be used together. Please provide only one.", error_output)
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_with_env_using_parameters(self):
         command_list = self.get_command_list(
             "EchoEnvWithParameters",
@@ -180,6 +191,7 @@ def test_invoke_with_env_using_parameters(self):
         self.assertEqual(environ["Timeout"], "100")
         self.assertEqual(environ["MyRuntimeVersion"], "v0")
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_with_env_using_parameters_with_custom_region(self):
         custom_region = "my-custom-region"
 
@@ -194,6 +206,7 @@ def test_invoke_with_env_using_parameters_with_custom_region(self):
 
         self.assertEqual(environ["Region"], custom_region)
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_with_env_with_aws_creds(self):
         custom_region = "my-custom-region"
         key = "key"
@@ -222,6 +235,7 @@ def test_invoke_with_env_with_aws_creds(self):
         self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], secret)
         self.assertEqual(environ["AWS_SESSION_TOKEN"], session)
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_invoke_with_docker_network_of_host(self):
         command_list = self.get_command_list(
             "HelloWorldServerlessFunction",
@@ -235,6 +249,7 @@ def test_invoke_with_docker_network_of_host(self):
 
         self.assertEqual(return_code, 0)
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     @skipIf(IS_WINDOWS, "The test hangs on Windows due to trying to attach to a non-existing network")
     def test_invoke_with_docker_network_of_host_in_env_var(self):
         command_list = self.get_command_list(
@@ -250,6 +265,7 @@ def test_invoke_with_docker_network_of_host_in_env_var(self):
 
         self.assertIn('Not Found ("network non-existing-network not found")', process_stderr.decode("utf-8"))
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_sam_template_file_env_var_set(self):
         command_list = self.get_command_list("HelloWorldFunctionInNonDefaultTemplate", event_path=self.event_path)
 
@@ -263,6 +279,7 @@ def test_sam_template_file_env_var_set(self):
 
         self.assertEqual(process_stdout.decode("utf-8"), '"Hello world"')
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_skip_pull_image_in_env_var(self):
         docker.from_env().api.pull("lambci/lambda:python3.6")
 
@@ -288,6 +305,7 @@ def setUp(self):
     def tearDown(self):
         shutil.rmtree(self.config_dir, ignore_errors=True)
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_existing_env_variables_precedence_over_profiles(self):
         profile = "default"
         custom_config = self._create_config_file(profile)
@@ -322,6 +340,7 @@ def test_existing_env_variables_precedence_over_profiles(self):
         self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], "priority_secret_key_id")
         self.assertEqual(environ["AWS_SESSION_TOKEN"], "priority_secret_token")
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_default_profile_with_custom_configs(self):
         profile = "default"
         custom_config = self._create_config_file(profile)
@@ -353,6 +372,7 @@ def test_default_profile_with_custom_configs(self):
         self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], "shhhhhthisisasecret")
         self.assertEqual(environ["AWS_SESSION_TOKEN"], "sessiontoken")
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_custom_profile_with_custom_configs(self):
         custom_config = self._create_config_file("custom")
         custom_cred = self._create_cred_file("custom")
@@ -383,6 +403,7 @@ def test_custom_profile_with_custom_configs(self):
         self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], "shhhhhthisisasecret")
         self.assertEqual(environ["AWS_SESSION_TOKEN"], "sessiontoken")
 
+    @pytest.mark.timeout(timeout=300, method="thread")
     def test_custom_profile_through_envrionment_variables(self):
         # When using a custom profile in a custom location, you need both the config
         # and credential file otherwise we fail to find a region or the profile (depending