Merge pull request #809 from awslabs/develop

chore: Release v0.8.0

.travis.yml

@@ -9,10 +9,6 @@ python:
   - "2.7"
   - "3.6"
 
-env:
-  global:
-    - AWS_DEFAULT_REGION=us-east-1
-
 # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs
 matrix:
   include:

DEVELOPMENT_GUIDE.rst

@@ -154,4 +154,4 @@ document to proceed with implementation.
 .. _tox: http://tox.readthedocs.io/en/latest/
 .. _numpy docstring: https://numpydoc.readthedocs.io/en/latest/format.html
 .. _pipenv: https://docs.pipenv.org/
-.. _design document template: ./design/_template.rst
+.. _design document template: ./designs/_template.rst

requirements/base.txt

@@ -3,12 +3,13 @@ chevron~=0.12
 click~=6.7
 enum34~=1.1.6; python_version<"3.4"
 Flask~=1.0.2
-boto3~=1.5
+boto3~=1.9, >=1.9.56
 PyYAML~=3.12
 cookiecutter~=1.6.0
-aws-sam-translator==1.8.0
+aws-sam-translator==1.9.0
 docker>=3.3.0
 dateparser~=0.7
 python-dateutil~=2.6
 pathlib2~=2.3.2; python_version<"3.4"
-aws_lambda_builders==0.0.2
+requests~=2.20.0
+aws_lambda_builders==0.0.3

requirements/dev.txt

@@ -2,6 +2,8 @@ coverage==4.3.4
 flake8==3.3.0
 tox==2.2.1
 pytest-cov==2.4.0
+# astroid > 2.0.4 is not compatible with pylint1.7
+astroid>=1.5.8,<2.1.0
 pylint==1.7.2
 
 # Test requirements

samcli/__init__.py

@@ -2,4 +2,4 @@
 SAM CLI version
 """
 
-__version__ = '0.7.0'
+__version__ = '0.8.0'

samcli/cli/options.py

@@ -41,6 +41,7 @@ def callback(ctx, param, value):
     return click.option('--region',
                         expose_value=False,
                         help='Set the AWS Region of the service (e.g. us-east-1).',
+                        default='us-east-1',
                         callback=callback)(f)
 
 

samcli/commands/_utils/options.py

@@ -96,7 +96,8 @@ def docker_click_options():
         click.option('--skip-pull-image',
                      is_flag=True,
                      help="Specify whether CLI should skip pulling down the latest Docker image for Lambda runtime.",
-                     envvar="SAM_SKIP_PULL_IMAGE"),
+                     envvar="SAM_SKIP_PULL_IMAGE",
+                     default=False),
 
         click.option('--docker-network',
                      envvar="SAM_DOCKER_NETWORK",

samcli/commands/_utils/template.py

@@ -2,22 +2,45 @@
 Utilities to manipulate template
 """
 
+import os
+import six
 import yaml
 
 try:
     import pathlib
 except ImportError:
     import pathlib2 as pathlib
 
-from samcli.yamlhelper import yaml_parse
+from samcli.yamlhelper import yaml_parse, yaml_dump
+
+
+_RESOURCES_WITH_LOCAL_PATHS = {
+    "AWS::Serverless::Function": ["CodeUri"],
+    "AWS::Serverless::Api": ["DefinitionUri"],
+    "AWS::AppSync::GraphQLSchema": ["DefinitionS3Location"],
+    "AWS::AppSync::Resolver": ["RequestMappingTemplateS3Location", "ResponseMappingTemplateS3Location"],
+    "AWS::Lambda::Function": ["Code"],
+    "AWS::ApiGateway::RestApi": ["BodyS3Location"],
+    "AWS::ElasticBeanstalk::ApplicationVersion": ["SourceBundle"],
+    "AWS::CloudFormation::Stack": ["TemplateURL"],
+    "AWS::Serverless::Application": ["Location"],
+    "AWS::Lambda::LayerVersion": ["Content"],
+    "AWS::Serverless::LayerVersion": ["ContentUri"]
+}
 
 
 def get_template_data(template_file):
     """
     Read the template file, parse it as JSON/YAML and return the template as a dictionary.
 
-    :param string template_file: Path to the template to read
-    :return dict: Template data as a dictionary
+    Parameters
+    ----------
+    template_file : string
+        Path to the template to read
+
+    Returns
+    -------
+    Template data as a dictionary
     """
 
     if not pathlib.Path(template_file).exists():
@@ -28,3 +51,167 @@ def get_template_data(template_file):
             return yaml_parse(fp.read())
         except (ValueError, yaml.YAMLError) as ex:
             raise ValueError("Failed to parse template: {}".format(str(ex)))
+
+
+def move_template(src_template_path,
+                  dest_template_path,
+                  template_dict):
+    """
+    Move the SAM/CloudFormation template from ``src_template_path`` to ``dest_template_path``. For convenience, this
+    method accepts a dictionary of template data ``template_dict`` that will be written to the destination instead of
+    reading from the source file.
+
+    SAM/CloudFormation template can contain certain properties whose value is a relative path to a local file/folder.
+    This path is always relative to the template's location. Before writing the template to ``dest_template_path`,
+    we will update these paths to be relative to the new location.
+
+    This methods updates resource properties supported by ``aws cloudformation package`` command:
+    https://docs.aws.amazon.com/cli/latest/reference/cloudformation/package.html
+
+    You must use this method if you are reading a template from one location, modifying it, and writing it back to a
+    different location.
+
+    Parameters
+    ----------
+    src_template_path : str
+        Path to the original location of the template
+
+    dest_template_path : str
+        Path to the destination location where updated template should be written to
+
+    template_dict : dict
+        Dictionary containing template contents. This dictionary will be updated & written to ``dest`` location.
+    """
+
+    original_root = os.path.dirname(src_template_path)
+    new_root = os.path.dirname(dest_template_path)
+
+    # Next up, we will be writing the template to a different location. Before doing so, we should
+    # update any relative paths in the template to be relative to the new location.
+    modified_template = _update_relative_paths(template_dict,
+                                               original_root,
+                                               new_root)
+
+    with open(dest_template_path, "w") as fp:
+        fp.write(yaml_dump(modified_template))
+
+
+def _update_relative_paths(template_dict,
+                           original_root,
+                           new_root):
+    """
+    SAM/CloudFormation template can contain certain properties whose value is a relative path to a local file/folder.
+    This path is usually relative to the template's location. If the template is being moved from original location
+    ``original_root`` to new location ``new_root``, use this method to update these paths to be
+    relative to ``new_root``.
+
+    After this method is complete, it is safe to write the template to ``new_root`` without
+    breaking any relative paths.
+
+    This methods updates resource properties supported by ``aws cloudformation package`` command:
+    https://docs.aws.amazon.com/cli/latest/reference/cloudformation/package.html
+
+    If a property is either an absolute path or a S3 URI, this method will not update them.
+
+
+    Parameters
+    ----------
+    template_dict : dict
+        Dictionary containing template contents. This dictionary will be updated & written to ``dest`` location.
+
+    original_root : str
+        Path to the directory where all paths were originally set relative to. This is usually the directory
+        containing the template originally
+
+    new_root : str
+        Path to the new directory that all paths set relative to after this method completes.
+
+    Returns
+    -------
+    Updated dictionary
+
+    """
+
+    for _, resource in template_dict.get("Resources", {}).items():
+        resource_type = resource.get("Type")
+
+        if resource_type not in _RESOURCES_WITH_LOCAL_PATHS:
+            # Unknown resource. Skipping
+            continue
+
+        for path_prop_name in _RESOURCES_WITH_LOCAL_PATHS[resource_type]:
+            properties = resource.get("Properties", {})
+            path = properties.get(path_prop_name)
+
+            updated_path = _resolve_relative_to(path, original_root, new_root)
+            if not updated_path:
+                # This path does not need to get updated
+                continue
+
+            properties[path_prop_name] = updated_path
+
+    # AWS::Includes can be anywhere within the template dictionary. Hence we need to recurse through the
+    # dictionary in a separate method to find and update relative paths in there
+    template_dict = _update_aws_include_relative_path(template_dict, original_root, new_root)
+
+    return template_dict
+
+
+def _update_aws_include_relative_path(template_dict, original_root, new_root):
+    """
+    Update relative paths in "AWS::Include" directive. This directive can be present at any part of the template,
+    and not just within resources.
+    """
+
+    for key, val in template_dict.items():
+        if key == "Fn::Transform":
+            if isinstance(val, dict) and val.get("Name") == "AWS::Include":
+                path = val.get("Parameters", {}).get("Location", {})
+                updated_path = _resolve_relative_to(path, original_root, new_root)
+                if not updated_path:
+                    # This path does not need to get updated
+                    continue
+
+                val["Parameters"]["Location"] = updated_path
+
+        # Recurse through all dictionary values
+        elif isinstance(val, dict):
+            _update_aws_include_relative_path(val, original_root, new_root)
+        elif isinstance(val, list):
+            for item in val:
+                if isinstance(item, dict):
+                    _update_aws_include_relative_path(item, original_root, new_root)
+
+    return template_dict
+
+
+def _resolve_relative_to(path, original_root, new_root):
+    """
+    If the given ``path`` is a relative path, then assume it is relative to ``original_root``. This method will
+    update the path to be resolve it relative to ``new_root`` and return.
+
+    Examples
+    -------
+        # Assume a file called template.txt at location /tmp/original/root/template.txt expressed as relative path
+        # We are trying to update it to be relative to /tmp/new/root instead of the /tmp/original/root
+        >>> result = _resolve_relative_to("template.txt",  \
+                                          "/tmp/original/root", \
+                                          "/tmp/new/root")
+        >>> result
+        ../../original/root/template.txt
+
+    Returns
+    -------
+    Updated path if the given path is a relative path. None, if the path is not a relative path.
+    """
+
+    if not isinstance(path, six.string_types) \
+            or path.startswith("s3://") \
+            or os.path.isabs(path):
+        # Value is definitely NOT a relative path. It is either a S3 URi or Absolute path or not a string at all
+        return None
+
+    # Value is definitely a relative path. Change it relative to the destination directory
+    return os.path.relpath(
+        os.path.normpath(os.path.join(original_root, path)),  # Absolute original path w.r.t ``original_root``
+        new_root)  # Resolve the original path with respect to ``new_root``

samcli/commands/build/build_context.py

@@ -117,6 +117,10 @@ def use_container(self):
     def output_template_path(self):
         return os.path.join(self._build_dir, "template.yaml")
 
+    @property
+    def original_template_path(self):
+        return os.path.abspath(self._template_file)
+
     @property
     def manifest_path_override(self):
         if self._manifest_path:

samcli/commands/build/command.py

@@ -7,13 +7,13 @@
 import click
 
 from samcli.commands.exceptions import UserException
-from samcli.yamlhelper import yaml_dump
 from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options
 from samcli.commands._utils.options import template_option_without_build, docker_common_options, \
     parameter_override_option
 from samcli.commands.build.build_context import BuildContext
 from samcli.lib.build.app_builder import ApplicationBuilder, UnsupportedRuntimeException, \
     BuildError, UnsupportedBuilderLibraryVersionError
+from samcli.commands._utils.template import move_template
 
 LOG = logging.getLogger(__name__)
 
@@ -32,6 +32,7 @@
 ------------------
 1. Python2.7\n
 2. Python3.6\n
+3. Python3.7\n
 \b
 Examples
 --------
@@ -130,11 +131,12 @@ def do_cli(template,  # pylint: disable=too-many-locals
         try:
             artifacts = builder.build()
             modified_template = builder.update_template(ctx.template_dict,
-                                                        ctx.output_template_path,
+                                                        ctx.original_template_path,
                                                         artifacts)
 
-            with open(ctx.output_template_path, "w") as fp:
-                fp.write(yaml_dump(modified_template))
+            move_template(ctx.original_template_path,
+                          ctx.output_template_path,
+                          modified_template)
 
             click.secho("\nBuild Succeeded", fg="green")
 

samcli/commands/deploy/__init__.py

@@ -6,6 +6,7 @@
 
 from samcli.cli.main import pass_context, common_options
 from samcli.lib.samlib.cloudformation_command import execute_command
+from samcli.commands.exceptions import UserException
 
 
 SHORT_HELP = "Deploy an AWS SAM application. This is an alias for 'aws cloudformation deploy'."
@@ -23,4 +24,7 @@ def cli(ctx, args):
 
 
 def do_cli(args):
-    execute_command("deploy", args, template_file=None)
+    try:
+        execute_command("deploy", args, template_file=None)
+    except OSError as ex:
+        raise UserException(str(ex))