Merge pull request #1598 from awslabs/develop

Release v0.34.0

requirements/base.txt

@@ -5,7 +5,7 @@ boto3~=1.9, >=1.9.56
 jmespath~=0.9.4
 PyYAML~=5.1
 cookiecutter~=1.6.0
-aws-sam-translator==1.15.1
+aws-sam-translator==1.16.0
 docker~=4.0
 dateparser~=0.7
 python-dateutil~=2.6, <2.8.1

samcli/__init__.py

@@ -2,4 +2,4 @@
 SAM CLI version
 """
 
-__version__ = "0.33.1"
+__version__ = "0.34.0"

samcli/commands/_utils/options.py

@@ -72,7 +72,7 @@ def guided_deploy_stack_name(ctx, param, provided_value):
         raise click.BadOptionUsage(
             option_name=param.name,
             ctx=ctx,
-            message="Missing option '--stack-name', 'sam deploy –guided' can "
+            message="Missing option '--stack-name', 'sam deploy --guided' can "
             "be used to provide and save needed parameters for future deploys.",
         )
 

samcli/commands/deploy/command.py

@@ -7,7 +7,7 @@
 import click
 from click.types import FuncParamType
 
-from samcli.lib.utils import temp_file_utils
+from samcli.lib.utils import osutils
 from samcli.cli.cli_config_file import configuration_option, TomlProvider
 from samcli.cli.context import get_cmd_names
 from samcli.cli.main import pass_context, common_options, aws_creds_options
@@ -113,7 +113,8 @@
     "executing the change set.",
 )
 @click.option(
-    "--fail-on-empty-changeset",
+    "--fail-on-empty-changeset/--no-fail-on-empty-changeset",
+    default=True,
     required=False,
     is_flag=True,
     help="Specify  if  the CLI should return a non-zero exit code if there are no"
@@ -256,7 +257,7 @@ def do_cli(
         confirm_changeset=changeset_decision if guided else confirm_changeset,
     )
 
-    with temp_file_utils.tempfile_platform_independent() as output_template_file:
+    with osutils.tempfile_platform_independent() as output_template_file:
 
         with PackageContext(
             template_file=template_file,

samcli/commands/init/init_generator.py

@@ -5,11 +5,11 @@
 
 from samcli.commands.exceptions import UserException
 from samcli.local.init import generate_project
-from samcli.local.init.exceptions import GenerateProjectFailedError
+from samcli.local.init.exceptions import GenerateProjectFailedError, ArbitraryProjectDownloadFailed
 
 
 def do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context):
     try:
         generate_project(location, runtime, dependency_manager, output_dir, name, no_input, extra_context)
-    except GenerateProjectFailedError as e:
+    except (GenerateProjectFailedError, ArbitraryProjectDownloadFailed) as e:
         raise UserException(str(e))

samcli/lib/utils/osutils.py

@@ -6,10 +6,15 @@
 import os
 import shutil
 import tempfile
+import logging
+import contextlib
 
 from contextlib import contextmanager
 
 
+LOG = logging.getLogger(__name__)
+
+
 @contextmanager
 def mkdir_temp(mode=0o755):
     """
@@ -62,3 +67,71 @@ def stderr():
         Byte stream of stderr
     """
     return sys.stderr.buffer
+
+
+def remove(path):
+    if path:
+        try:
+            os.remove(path)
+        except OSError:
+            pass
+
+
+@contextlib.contextmanager
+def tempfile_platform_independent():
+    # NOTE(TheSriram): Setting delete=False is specific to windows.
+    # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
+    _tempfile = tempfile.NamedTemporaryFile(delete=False)
+    try:
+        yield _tempfile
+    finally:
+        _tempfile.close()
+        remove(_tempfile.name)
+
+
+# NOTE: Py3.8 or higher has a ``dir_exist_ok=True`` parameter to provide this functionality.
+#       This method can be removed if we stop supporting Py37
+def copytree(source, destination, ignore=None):
+    """
+    Similar to shutil.copytree except that it removes the limitation that the destination directory should
+    be present.
+    :type source: str
+    :param source:
+        Path to the source folder to copy
+    :type destination: str
+    :param destination:
+        Path to destination folder
+    :type ignore: function
+    :param ignore:
+        A function that returns a set of file names to ignore, given a list of available file names. Similar to the
+        ``ignore`` property of ``shutils.copytree`` method
+    """
+
+    if not os.path.exists(destination):
+        os.makedirs(destination)
+
+        try:
+            # Let's try to copy the directory metadata from source to destination
+            shutil.copystat(source, destination)
+        except OSError as ex:
+            # Can't copy file access times in Windows
+            LOG.debug("Unable to copy file access times from %s to %s", source, destination, exc_info=ex)
+
+    names = os.listdir(source)
+    if ignore is not None:
+        ignored_names = ignore(source, names)
+    else:
+        ignored_names = set()
+
+    for name in names:
+        # Skip ignored names
+        if name in ignored_names:
+            continue
+
+        new_source = os.path.join(source, name)
+        new_destination = os.path.join(destination, name)
+
+        if os.path.isdir(new_source):
+            copytree(new_source, new_destination, ignore=ignore)
+        else:
+            shutil.copy2(new_source, new_destination)

samcli/lib/utils/temp_file_utils.py

@@ -2,25 +2,4 @@
 Helper functions for temporary files
 """
 import os
-import contextlib
 import tempfile
-
-
-def remove(path):
-    if path:
-        try:
-            os.remove(path)
-        except OSError:
-            pass
-
-
-@contextlib.contextmanager
-def tempfile_platform_independent():
-    # NOTE(TheSriram): Setting delete=False is specific to windows.
-    # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
-    _tempfile = tempfile.NamedTemporaryFile(delete=False)
-    try:
-        yield _tempfile
-    finally:
-        _tempfile.close()
-        remove(_tempfile.name)

samcli/local/init/__init__.py

@@ -4,11 +4,14 @@
 import itertools
 import logging
 
-from cookiecutter.exceptions import CookiecutterException
+from pathlib import Path
+
+from cookiecutter.exceptions import CookiecutterException, RepositoryNotFound
 from cookiecutter.main import cookiecutter
 
 from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING
 from samcli.local.init.exceptions import GenerateProjectFailedError
+from .arbitrary_project import generate_non_cookiecutter_project
 
 LOG = logging.getLogger(__name__)
 
@@ -76,5 +79,15 @@ def generate_project(
     try:
         LOG.debug("Baking a new template with cookiecutter with all parameters")
         cookiecutter(**params)
+    except RepositoryNotFound as e:
+        # cookiecutter.json is not found in the template. Let's just clone it directly without using cookiecutter
+        # and call it done.
+        LOG.debug(
+            "Unable to find cookiecutter.json in the project. Downloading it directly without treating "
+            "it as a cookiecutter template"
+        )
+        project_output_dir = str(Path(output_dir, name)) if name else output_dir
+        generate_non_cookiecutter_project(location=params["template"], output_dir=project_output_dir)
+
     except CookiecutterException as e:
         raise GenerateProjectFailedError(project=name, provider_error=e)

samcli/local/init/arbitrary_project.py

@@ -0,0 +1,111 @@
+"""
+Initialize an arbitrary project
+"""
+
+import functools
+import shutil
+import logging
+
+from pathlib import Path
+from cookiecutter import repository
+from cookiecutter import exceptions
+from cookiecutter import config
+
+from samcli.lib.utils import osutils
+from .exceptions import ArbitraryProjectDownloadFailed
+
+
+LOG = logging.getLogger(__name__)
+
+
+BAD_LOCATION_ERROR_MSG = (
+    "Please verify your location. The following types of location are supported:"
+    "\n\n* Github: gh:user/repo (or) https://github.com/user/repo (or) git@github.com:user/repo.git"
+    "\n          For Git repositories, you must use location of the root of the repository."
+    "\n\n* Mercurial: hg+ssh://hg@bitbucket.org/repo"
+    "\n\n* Http(s): https://example.com/code.zip"
+    "\n\n* Local Path: /path/to/code.zip"
+)
+
+
+def generate_non_cookiecutter_project(location, output_dir):
+    """
+    Uses Cookiecutter APIs to download a project at given ``location`` to the ``output_dir``.
+    This does *not* run cookiecutter on the downloaded project.
+
+    Parameters
+    ----------
+    location : str
+        Path to where the project is. This supports all formats of location cookiecutter supports
+        (ex: zip, git, ssh, hg, local zipfile)
+
+        NOTE: This value *cannot* be a local directory. We didn't see a value in simply copying the directory
+        contents to ``output_dir`` without any processing.
+
+    output_dir : str
+        Directory where the project should be downloaded to
+
+    Returns
+    -------
+    str
+        Name of the directory where the project was downloaded to.
+
+    Raises
+    ------
+    cookiecutter.exception.CookiecutterException if download failed for some reason
+    """
+
+    LOG.debug("Downloading project from %s to %s", location, output_dir)
+
+    # Don't prompt ever
+    no_input = True
+
+    # Expand abbreviations in URL such as gh:awslabs/aws-sam-cli
+    location = repository.expand_abbreviations(location, config.BUILTIN_ABBREVIATIONS)
+
+    # If this is a zip file, download and unzip into output directory
+    if repository.is_zip_file(location):
+        LOG.debug("%s location is a zip file", location)
+        download_fn = functools.partial(
+            repository.unzip, zip_uri=location, is_url=repository.is_repo_url(location), no_input=no_input
+        )
+
+    # Else, treat it as a git/hg/ssh URL and try to clone
+    elif repository.is_repo_url(location):
+        LOG.debug("%s location is a source control repository", location)
+        download_fn = functools.partial(repository.clone, repo_url=location, no_input=no_input)
+
+    else:
+        raise ArbitraryProjectDownloadFailed(msg=BAD_LOCATION_ERROR_MSG)
+
+    try:
+        return _download_and_copy(download_fn, output_dir)
+    except exceptions.RepositoryNotFound:
+        # Download failed because the zip or the repository was not found
+        raise ArbitraryProjectDownloadFailed(msg=BAD_LOCATION_ERROR_MSG)
+
+
+def _download_and_copy(download_fn, output_dir):
+    """
+    Runs the download function to download files into a temporary directory and then copy the files over to
+    the ``output_dir``
+
+    Parameters
+    ----------
+    download_fn : function
+        Method to be called to download. It needs to accept a parameter called `clone_to_dir`. This will be
+        set to the temporary directory
+
+    output_dir : str
+        Path to the directory where files will be copied to
+
+    Returns
+    -------
+    output_dir
+    """
+
+    with osutils.mkdir_temp() as tempdir:
+        downloaded_dir = download_fn(clone_to_dir=tempdir)
+        osutils.copytree(downloaded_dir, output_dir, ignore=shutil.ignore_patterns("*.git"))
+
+    return output_dir

samcli/local/init/exceptions.py

@@ -13,4 +13,8 @@ def __init__(self, **kwargs):
 
 
 class GenerateProjectFailedError(InitErrorException):
-    fmt = "An error occurred while generating this {project}: {provider_error}"
+    fmt = "An error occurred while generating this project {project}: {provider_error}"
+
+
+class ArbitraryProjectDownloadFailed(InitErrorException):
+    fmt = "{msg}"

tests/functional/commands/validate/lib/models/function_with_batch_window.yaml → tests/functional/commands/validate/lib/models/function_with_event_source_mapping.yaml

@@ -1,6 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
 Transform: AWS::Serverless-2016-10-31
-Description: EventSourceMapping example with MaximumBatchingWindowInSeconds property
 
 Parameters:
   MyBatchingWindowParam:
@@ -23,6 +22,9 @@ Resources:
             }
         }
       Runtime: nodejs8.10
+      Policies:
+        - SQSSendMessagePolicy:
+            QueueName: !GetAtt MySqsQueue.QueueName
       Events:
         Stream:
           Type: Kinesis
@@ -42,7 +44,14 @@ Resources:
             Stream: !GetAtt DynamoDBTable.StreamArn
             BatchSize: 100
             MaximumBatchingWindowInSeconds: !Ref MyBatchingWindowParam
+            ParallelizationFactor: 8
+            MaximumRetryAttempts: 100
+            BisectBatchOnFunctionError: true
+            MaximumRecordAgeInSeconds: 86400
             StartingPosition: TRIM_HORIZON
+            DestinationConfig:
+              OnFailure:
+                Destination: !GetAtt MySqsQueue.Arn
 
   KinesisStream:
     Type: AWS::Kinesis::Stream
@@ -66,4 +75,7 @@ Resources:
         ReadCapacityUnits: 5
         WriteCapacityUnits: 5
       StreamSpecification:
-        StreamViewType: NEW_IMAGE
+        StreamViewType: NEW_IMAGE
+
+  MySqsQueue:
+    Type: AWS::SQS::Queue

tests/functional/commands/validate/lib/test_sam_template_validator.py

@@ -90,7 +90,6 @@ class TestValidate(TestCase):
         ("tests/functional/commands/validate/lib/models/function_with_alias.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_alias_and_event_sources.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_alias_intrinsics.yaml"),
-        ("tests/functional/commands/validate/lib/models/function_with_batch_window.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_condition.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_conditional_managed_policy.yaml"),
         (
@@ -113,6 +112,7 @@ class TestValidate(TestCase):
         ),
         ("tests/functional/commands/validate/lib/models/function_with_disabled_deployment_preference.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_dlq.yaml"),
+        ("tests/functional/commands/validate/lib/models/function_with_event_source_mapping.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_global_layers.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_kmskeyarn.yaml"),
         ("tests/functional/commands/validate/lib/models/function_with_layers.yaml"),