Merge pull request #1520 from awslabs/develop

Release v0.31.1

Makefile

@@ -20,6 +20,10 @@ func-test:
 	@echo Telemetry Status: $(SAM_CLI_TELEMETRY)
 	pytest --cov samcli.local --cov samcli.commands.local --cov-report term-missing tests/functional/commands/validate tests/functional/commands/cli/test_global_config.py
 
+regres-test:
+	@echo Telemetry Status: $(SAM_CLI_TELEMETRY)
+	SAM_CLI_DEV=1 pytest tests/regression
+
 smoke-test:
 	# Smoke tests run in parallel
 	SAM_CLI_DEV=1 pytest -n 4 tests/smoke

appveyor-windows-build-java.yml → appveyor-windows-build-java-container.yml

@@ -27,20 +27,12 @@ environment:
   HOMEDRIVE: 'C:'
   HOMEPATH: 'C:\Users\appveyor'
 
-  matrix:
-    - optional_gate: true
-
-matrix:
-  allow_failures:
-    - optional_gate: true
-
 init:
   # Uncomment this for RDP
-  - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+  # - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
   - ps: gcim Win32_Processor | % { "$($_.NumberOfLogicalProcessors) logical CPUs" }
   - ps: gcim Win32_OperatingSystem | % { "$([int]($_.TotalVisibleMemorySize/1mb)) Gb" }
 
-
 install:
 
     # Make sure the temp directory exists for Python to use.
@@ -83,7 +75,8 @@ test_script:
     # Reactivate virtualenv before running tests
     - "venv\\Scripts\\activate"
     - "docker system prune -a -f"
-    - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k TestBuildCommand_Java"
+    - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java_in_container"
+
 
 # Uncomment for RDP
 # on_finish:

appveyor-windows-build-java-inprocess.yml

@@ -0,0 +1,58 @@
+version: 1.0.{build}
+image: Visual Studio 2017
+build: off
+
+environment:
+  AWS_DEFAULT_REGION: us-east-1
+  SAM_CLI_DEV: 1
+  APPVEYOR_CI_OVERRIDE: 1
+  GRADLE_OPTS: -Dorg.gradle.daemon=false
+
+  # MSI Installers only use Py3.6.6. It is sufficient to test with this version here.
+  PYTHON_HOME: "C:\\Python36-x64"
+  PYTHON_SCRIPTS: "C:\\Python36-x64\\Scripts"
+  PYTHON_EXE: "C:\\Python36-x64\\python.exe"
+  PYTHON_VERSION: '3.6.8'
+  PYTHON_ARCH: '64'
+
+install:
+
+    - "SET PATH=%PYTHON_HOME%;%PATH%"
+    - "echo %PYTHON_HOME%"
+    - "echo %PATH%"
+    - "python --version"
+
+    # Upgrade setuptools, wheel and virtualenv
+    - "python -m pip install --upgrade setuptools wheel virtualenv"
+
+    # Create new virtual environment with chosen python version and activate it
+    - "python -m virtualenv venv"
+    - "venv\\Scripts\\activate"
+    - "python --version"
+
+    # Actually install SAM CLI's dependencies
+    - "pip install -e \".[dev]\""
+
+    # setup Java, Maven and Gradle
+    - "refreshenv"
+    - "choco install maven -y --force"
+    - "refreshenv"
+    - "choco install gradle -y --force"
+    - "refreshenv"
+    - "java -version"
+    - "gradle -v"
+    - "mvn --version"
+
+    # setup Ruby dependencies
+    - "set PATH=C:\\Ruby25-x64\\bin;%PATH%"
+    - "gem --version"
+    - "gem install bundler -v 1.17.3"
+    - "bundler --version"
+
+    # Echo final Path
+    - "echo %PATH%"
+
+test_script:
+    # Reactivate virtualenv before running tests
+    - "venv\\Scripts\\activate"
+    - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java_in_process"

appveyor-windows-build-ruby.yml

@@ -27,13 +27,6 @@ environment:
   HOMEDRIVE: 'C:'
   HOMEPATH: 'C:\Users\appveyor'
 
-  matrix:
-    - optional_gate: true
-
-matrix:
-  allow_failures:
-    - optional_gate: true
-
 init:
   # Uncomment this for RDP
   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
@@ -81,7 +74,7 @@ test_script:
     # Reactivate virtualenv before running tests
     - "venv\\Scripts\\activate"
     - "docker system prune -a -f"
-    - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k TestBuildCommand_RubyFunctions"
+    - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_ruby_in_container"
 
 # Uncomment for RDP
 # on_finish:

requirements/base.txt

@@ -12,3 +12,5 @@ python-dateutil~=2.6, <2.8.1
 requests==2.22.0
 serverlessrepo==0.1.9
 aws_lambda_builders==0.5.0
+# https://github.com/mhammond/pywin32/issues/1439
+pywin32 < 226; sys_platform == 'win32'

samcli/__init__.py

@@ -2,4 +2,4 @@
 SAM CLI version
 """
 
-__version__ = "0.31.0"
+__version__ = "0.31.1"

samcli/commands/init/__init__.py

@@ -3,9 +3,12 @@
 Init command to scaffold a project app from a template
 """
 import logging
+import json
+from json import JSONDecodeError
 
 import click
 
+from samcli.commands.exceptions import UserException
 from samcli.cli.main import pass_context, common_options, global_cfg
 from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS
 from samcli.lib.telemetry.metrics import track_command
@@ -86,12 +89,32 @@
     default=False,
     help="Disable Cookiecutter prompting and accept default values defined template config",
 )
+@click.option(
+    "--extra_context",
+    default=None,
+    help="Override any custom parameters in the template's cookiecutter.json configuration e.g. "
+    ""
+    '{"customParam1": "customValue1", "customParam2":"customValue2"}'
+    """ """,
+    required=False,
+)
 @common_options
 @pass_context
 @track_command
-def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input):
+def cli(
+    ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input, extra_context
+):
     do_cli(
-        ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input
+        ctx,
+        no_interactive,
+        location,
+        runtime,
+        dependency_manager,
+        output_dir,
+        name,
+        app_template,
+        no_input,
+        extra_context,
     )  # pragma: no cover
 
 
@@ -106,9 +129,9 @@ def do_cli(
     name,
     app_template,
     no_input,
+    extra_context,
     auto_clone=True,
 ):
-    from samcli.commands.exceptions import UserException
     from samcli.commands.init.init_generator import do_generate
     from samcli.commands.init.init_templates import InitTemplates
     from samcli.commands.init.interactive_init_flow import do_interactive
@@ -126,12 +149,15 @@ def do_cli(
     # check for required parameters
     if location or (name and runtime and dependency_manager and app_template):
         # need to turn app_template into a location before we generate
-        extra_context = None
         if app_template:
             templates = InitTemplates(no_interactive, auto_clone)
             location = templates.location_from_app_template(runtime, dependency_manager, app_template)
             no_input = True
-            extra_context = {"project_name": name, "runtime": runtime}
+            default_context = {"project_name": name, "runtime": runtime}
+            if extra_context is None:
+                extra_context = default_context
+            else:
+                extra_context = _merge_extra_context(default_context, extra_context)
         if not output_dir:
             output_dir = "."
         do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context)
@@ -149,3 +175,13 @@ def do_cli(
     else:
         # proceed to interactive state machine, which will call do_generate
         do_interactive(location, runtime, dependency_manager, output_dir, name, app_template, no_input)
+
+
+def _merge_extra_context(default_context, extra_context):
+    try:
+        extra_context_dict = json.loads(extra_context)
+    except JSONDecodeError:
+        raise UserException(
+            "Parse error reading the --extra-content parameter. The value of this parameter must be valid JSON."
+        )
+    return {**extra_context_dict, **default_context}

samcli/commands/local/lib/sam_function_provider.py

@@ -1,11 +1,9 @@
 """
 Class that provides functions from a given SAM template
 """
-
-import ast
 import logging
 
-from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn, InvalidSamTemplateException
+from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn
 from .exceptions import InvalidLayerReference
 from .provider import FunctionProvider, Function, LayerVersion
 from .sam_base_provider import SamBaseProvider
@@ -123,18 +121,11 @@ def _convert_sam_function_resource(name, resource_properties, layers):
 
         LOG.debug("Found Serverless function with name='%s' and CodeUri='%s'", name, codeuri)
 
-        timeout = resource_properties.get("Timeout")
-        if isinstance(timeout, str):
-            try:
-                timeout = ast.literal_eval(timeout)
-            except ValueError:
-                raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(timeout))
-
         return Function(
             name=name,
             runtime=resource_properties.get("Runtime"),
             memory=resource_properties.get("MemorySize"),
-            timeout=timeout,
+            timeout=resource_properties.get("Timeout"),
             handler=resource_properties.get("Handler"),
             codeuri=codeuri,
             environment=resource_properties.get("Environment"),

samcli/local/lambdafn/config.py

@@ -1,7 +1,7 @@
 """
 Lambda Function configuration data required by the runtime
 """
-
+from samcli.commands.local.cli_common.user_exceptions import InvalidSamTemplateException
 from .env_vars import EnvironmentVariables
 
 
@@ -44,8 +44,16 @@ def __init__(self, name, runtime, handler, code_abs_path, layers, memory=None, t
         self.code_abs_path = code_abs_path
         self.layers = layers
         self.memory = memory or self._DEFAULT_MEMORY
+
         self.timeout = timeout or self._DEFAULT_TIMEOUT_SECONDS
 
+        if not isinstance(self.timeout, int):
+            try:
+                self.timeout = int(self.timeout)
+
+            except (ValueError, TypeError):
+                raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(self.timeout))
+
         if not env_vars:
             env_vars = EnvironmentVariables(self.memory, self.timeout, self.handler)
 

tests/integration/buildcmd/test_build_cmd.py

@@ -5,6 +5,7 @@
 from unittest import skipIf
 from pathlib import Path
 from parameterized import parameterized
+import pytest
 
 from .build_integ_base import BuildIntegBase
 from tests.testing_utils import IS_WINDOWS, RUNNING_ON_CI, CI_OVERRIDE
@@ -201,8 +202,17 @@ class TestBuildCommand_RubyFunctions(BuildIntegBase):
 
     FUNCTION_LOGICAL_ID = "Function"
 
-    @parameterized.expand([("ruby2.5", False), ("ruby2.5", "use_container")])
-    def test_with_default_gemfile(self, runtime, use_container):
+    @pytest.mark.flaky(reruns=3)
+    @pytest.mark.timeout(timeout=300, method="thread")
+    @parameterized.expand([("ruby2.5")])
+    def test_building_ruby_in_container(self, runtime):
+        self._test_with_default_gemfile(runtime, "use_container")
+
+    @parameterized.expand([("ruby2.5")])
+    def test_building_ruby_in_process(self, runtime):
+        self._test_with_default_gemfile(runtime, False)
+
+    def _test_with_default_gemfile(self, runtime, use_container):
         overrides = {"Runtime": runtime, "CodeUri": "Ruby", "Handler": "ignored"}
         cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides)
 
@@ -277,21 +287,36 @@ class TestBuildCommand_Java(BuildIntegBase):
     WINDOWS_LINE_ENDING = b"\r\n"
     UNIX_LINE_ENDING = b"\n"
 
+    @pytest.mark.flaky(reruns=3)
+    @pytest.mark.timeout(timeout=300, method="thread")
+    @parameterized.expand(
+        [
+            ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
+            ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
+            ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
+            ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN),
+            ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
+        ]
+    )
+    def test_building_java_in_container(self, runtime, code_path, expected_files):
+        self._test_with_building_java(runtime, code_path, expected_files, "use_container")
+
     @parameterized.expand(
         [
-            ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, False),
-            ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, False),
-            ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, False),
-            ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, False),
-            ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, "use_container"),
-            ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, "use_container"),
-            ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, "use_container"),
-            ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, "use_container"),
+            ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
+            ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
+            ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
+            ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN),
+            ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE),
         ]
     )
-    def test_with_building_java(self, runtime, code_path, expected_files, use_container):
+    def test_building_java_in_process(self, runtime, code_path, expected_files):
+        self._test_with_building_java(runtime, code_path, expected_files, False)
+
+    def _test_with_building_java(self, runtime, code_path, expected_files, use_container):
         overrides = {"Runtime": runtime, "CodeUri": code_path, "Handler": "aws.example.Hello::myHandler"}
         cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides)
+        cmdlist += ["--skip-pull-image"]
         if code_path == self.USING_GRADLEW_PATH and use_container and IS_WINDOWS:
             self._change_to_unix_line_ending(os.path.join(self.test_data_path, self.USING_GRADLEW_PATH, "gradlew"))
 
@@ -313,12 +338,14 @@ def test_with_building_java(self, runtime, code_path, expected_files, use_contai
             ),
         )
 
-        expected = "Hello World"
-        self._verify_invoke_built_function(
-            self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected
-        )
+        # If we are testing in the container, invoke the function as well. Otherwise we cannot guarantee docker is on appveyor
+        if use_container:
+            expected = "Hello World"
+            self._verify_invoke_built_function(
+                self.built_template, self.FUNCTION_LOGICAL_ID, self._make_parameter_override_arg(overrides), expected
+            )
 
-        self.verify_docker_container_cleanedup(runtime)
+            self.verify_docker_container_cleanedup(runtime)
 
     def _verify_built_artifact(self, build_dir, function_logical_id, expected_files, expected_modules):