From 1d7df8fce701711963efc32507305c2fdd84a3aa Mon Sep 17 00:00:00 2001 From: Sanath Kumar Ramesh Date: Tue, 20 Nov 2018 14:05:56 -0800 Subject: [PATCH 1/5] fix: AWS CLI on windows can be called aws.cmd or aws.exe (#786) --- samcli/commands/deploy/__init__.py | 6 +- samcli/commands/package/__init__.py | 6 +- samcli/lib/samlib/cloudformation_command.py | 24 +++- .../lib/samlib/test_cloudformation_command.py | 120 ++++++++++++++---- 4 files changed, 127 insertions(+), 29 deletions(-) diff --git a/samcli/commands/deploy/__init__.py b/samcli/commands/deploy/__init__.py index dd2173bebd..a16380d684 100644 --- a/samcli/commands/deploy/__init__.py +++ b/samcli/commands/deploy/__init__.py @@ -6,6 +6,7 @@ from samcli.cli.main import pass_context, common_options from samcli.lib.samlib.cloudformation_command import execute_command +from samcli.commands.exceptions import UserException SHORT_HELP = "Deploy an AWS SAM application. This is an alias for 'aws cloudformation deploy'." @@ -23,4 +24,7 @@ def cli(ctx, args): def do_cli(args): - execute_command("deploy", args, template_file=None) + try: + execute_command("deploy", args, template_file=None) + except OSError as ex: + raise UserException(str(ex)) diff --git a/samcli/commands/package/__init__.py b/samcli/commands/package/__init__.py index 32ed7fbd41..ff16662d09 100644 --- a/samcli/commands/package/__init__.py +++ b/samcli/commands/package/__init__.py @@ -8,6 +8,7 @@ from samcli.cli.main import pass_context, common_options from samcli.commands._utils.options import get_or_default_template_file_name, _TEMPLATE_OPTION_DEFAULT_VALUE from samcli.lib.samlib.cloudformation_command import execute_command +from samcli.commands.exceptions import UserException SHORT_HELP = "Package an AWS SAM application. This is an alias for 'aws cloudformation package'." @@ -31,4 +32,7 @@ def cli(ctx, args, template_file): def do_cli(args, template_file): - execute_command("package", args, template_file) + try: + execute_command("package", args, template_file) + except OSError as ex: + raise UserException(str(ex)) diff --git a/samcli/lib/samlib/cloudformation_command.py b/samcli/lib/samlib/cloudformation_command.py index 94cdb4bb86..93914827ad 100644 --- a/samcli/lib/samlib/cloudformation_command.py +++ b/samcli/lib/samlib/cloudformation_command.py @@ -13,7 +13,7 @@ def execute_command(command, args, template_file): LOG.debug("%s command is called", command) try: - aws_cmd = 'aws' if platform.system().lower() != 'windows' else 'aws.cmd' + aws_cmd = find_executable("aws") args = list(args) if template_file: @@ -26,3 +26,25 @@ def execute_command(command, args, template_file): # Underlying aws command will print the exception to the user LOG.debug("Exception: %s", e) sys.exit(e.returncode) + + +def find_executable(execname): + + if platform.system().lower() == 'windows': + options = [ + "{}.cmd".format(execname), + "{}.exe".format(execname), + execname + ] + else: + options = [execname] + + for name in options: + try: + subprocess.Popen([name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # No exception. Let's pick this + return name + except OSError as ex: + LOG.debug("Unable to find executable %s", name, exc_info=ex) + + raise OSError("Unable to find AWS CLI installation under following names: {}".format(options)) diff --git a/tests/unit/lib/samlib/test_cloudformation_command.py b/tests/unit/lib/samlib/test_cloudformation_command.py index 5be0fa4975..7aa3248529 100644 --- a/tests/unit/lib/samlib/test_cloudformation_command.py +++ b/tests/unit/lib/samlib/test_cloudformation_command.py @@ -2,52 +2,120 @@ Tests Deploy CLI """ -from subprocess import CalledProcessError +from subprocess import CalledProcessError, PIPE from unittest import TestCase -from mock import patch +from mock import patch, call -from samcli.lib.samlib.cloudformation_command import execute_command +from samcli.lib.samlib.cloudformation_command import execute_command, find_executable -class TestCli(TestCase): +class TestExecuteCommand(TestCase): def setUp(self): self.args = ("--arg1", "value1", "different args", "more") @patch("subprocess.check_call") - @patch("platform.system") - def test_must_add_template_file(self, platform_system_mock, check_call_mock): - platform_system_mock.return_value = "Linux" + @patch("samcli.lib.samlib.cloudformation_command.find_executable") + def test_must_add_template_file(self, find_executable_mock, check_call_mock): + find_executable_mock.return_value = "mycmd" check_call_mock.return_value = True execute_command("command", self.args, "/path/to/template") - check_call_mock.assert_called_with(["aws", "cloudformation", "command"] + + check_call_mock.assert_called_with(["mycmd", "cloudformation", "command"] + ["--arg1", "value1", "different args", "more", "--template-file", "/path/to/template"]) + @patch("sys.exit") @patch("subprocess.check_call") + @patch("samcli.lib.samlib.cloudformation_command.find_executable") + def test_command_must_exit_with_status_code(self, find_executable_mock, check_call_mock, exit_mock): + find_executable_mock.return_value = "mycmd" + check_call_mock.side_effect = CalledProcessError(2, "Error") + exit_mock.return_value = True + execute_command("command", self.args, None) + exit_mock.assert_called_with(2) + + +class TestFindExecutable(TestCase): + + @patch("subprocess.Popen") @patch("platform.system") - def test_command_must_call_aws_linux(self, platform_system_mock, check_call_mock): + def test_must_use_raw_name(self, platform_system_mock, popen_mock): platform_system_mock.return_value = "Linux" - check_call_mock.return_value = True - execute_command("command", self.args, None) - check_call_mock.assert_called_with(["aws", "cloudformation", "command"] + list(self.args)) + execname = "foo" - @patch("subprocess.check_call") + find_executable(execname) + + self.assertEquals(popen_mock.mock_calls, [ + call([execname], stdout=PIPE, stderr=PIPE) + ]) + + @patch("subprocess.Popen") @patch("platform.system") - def test_command_must_call_aws_windows(self, platform_system_mock, check_call_mock): - platform_system_mock.return_value = "Windows" - check_call_mock.return_value = True - execute_command("command", self.args, None) - check_call_mock.assert_called_with(["aws.cmd", "cloudformation", "command"] + list(self.args)) + def test_must_use_name_with_cmd_extension_on_windows(self, platform_system_mock, popen_mock): + platform_system_mock.return_value = "windows" + execname = "foo" + expected = "foo.cmd" - @patch("sys.exit") - @patch("subprocess.check_call") + result = find_executable(execname) + self.assertEquals(result, expected) + + self.assertEquals(popen_mock.mock_calls, [ + call(["foo.cmd"], stdout=PIPE, stderr=PIPE) + ]) + + @patch("subprocess.Popen") @patch("platform.system") - def test_command_must_exit_with_status_code(self, platform_system_mock, check_call_mock, exit_mock): - platform_system_mock.return_value = "Any" - check_call_mock.side_effect = CalledProcessError(2, "Error") - exit_mock.return_value = True - execute_command("command", self.args, None) - exit_mock.assert_called_with(2) + def test_must_use_name_with_exe_extension_on_windows(self, platform_system_mock, popen_mock): + platform_system_mock.return_value = "windows" + execname = "foo" + expected = "foo.exe" + + popen_mock.side_effect = [OSError, "success"] # fail on .cmd extension + + result = find_executable(execname) + self.assertEquals(result, expected) + + self.assertEquals(popen_mock.mock_calls, [ + call(["foo.cmd"], stdout=PIPE, stderr=PIPE), + call(["foo.exe"], stdout=PIPE, stderr=PIPE) + ]) + + @patch("subprocess.Popen") + @patch("platform.system") + def test_must_use_name_with_no_extension_on_windows(self, platform_system_mock, popen_mock): + platform_system_mock.return_value = "windows" + execname = "foo" + expected = "foo" + + popen_mock.side_effect = [OSError, OSError, "success"] # fail on .cmd and .exe extension + + result = find_executable(execname) + self.assertEquals(result, expected) + + self.assertEquals(popen_mock.mock_calls, [ + call(["foo.cmd"], stdout=PIPE, stderr=PIPE), + call(["foo.exe"], stdout=PIPE, stderr=PIPE), + call(["foo"], stdout=PIPE, stderr=PIPE), + ]) + + @patch("subprocess.Popen") + @patch("platform.system") + def test_must_raise_error_if_executable_not_found(self, platform_system_mock, popen_mock): + platform_system_mock.return_value = "windows" + execname = "foo" + + popen_mock.side_effect = [OSError, OSError, OSError, "success"] # fail on all executable names + + with self.assertRaises(OSError) as ctx: + find_executable(execname) + + expected = "Unable to find AWS CLI installation under following names: {}".format(["foo.cmd", "foo.exe", "foo"]) + self.assertEquals(expected, str(ctx.exception)) + + self.assertEquals(popen_mock.mock_calls, [ + call(["foo.cmd"], stdout=PIPE, stderr=PIPE), + call(["foo.exe"], stdout=PIPE, stderr=PIPE), + call(["foo"], stdout=PIPE, stderr=PIPE), + ]) From b2fa6fe2e218db275e9cc44f95d1c24209c560a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?jos=C3=A9=20ferraris?= Date: Sat, 24 Nov 2018 00:02:28 +0100 Subject: [PATCH 2/5] fix: typo in development guide doc (#791) --- DEVELOPMENT_GUIDE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEVELOPMENT_GUIDE.rst b/DEVELOPMENT_GUIDE.rst index cc91676fb8..b1bc8e21f5 100644 --- a/DEVELOPMENT_GUIDE.rst +++ b/DEVELOPMENT_GUIDE.rst @@ -154,4 +154,4 @@ document to proceed with implementation. .. _tox: http://tox.readthedocs.io/en/latest/ .. _numpy docstring: https://numpydoc.readthedocs.io/en/latest/format.html .. _pipenv: https://docs.pipenv.org/ -.. _design document template: ./design/_template.rst +.. _design document template: ./designs/_template.rst From 58feafed6ee41074bf4eef0c0cfed1052e48fd6a Mon Sep 17 00:00:00 2001 From: Dobrianskyi Nikita <34073648+ndobryanskyy@users.noreply.github.com> Date: Mon, 26 Nov 2018 19:51:52 +0200 Subject: [PATCH 3/5] Fix astroid version By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. --- requirements/dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index 8a3fe34dbd..67ae56be74 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,6 +2,7 @@ coverage==4.3.4 flake8==3.3.0 tox==2.2.1 pytest-cov==2.4.0 +astroid>=1.5.8,<2.1.0 pylint==1.7.2 # Test requirements From a7e81cb91508c85ed0bd9d9fa55662f7a48abaf2 Mon Sep 17 00:00:00 2001 From: Dobrianskyi Nikita Date: Mon, 26 Nov 2018 20:07:04 +0200 Subject: [PATCH 4/5] Add explanations for version pinning --- requirements/dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index 67ae56be74..af480e2bf1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,6 +2,7 @@ coverage==4.3.4 flake8==3.3.0 tox==2.2.1 pytest-cov==2.4.0 +# astroid > 2.0.4 is not compatible with pylint1.7 astroid>=1.5.8,<2.1.0 pylint==1.7.2 From f88fb592e072314a4902e1f00489329a58c79519 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Thu, 29 Nov 2018 12:35:38 -0800 Subject: [PATCH 5/5] feat: Support for Lambda Layers, Ruby2.5, Python3.7, and Custom Runtimes (#808) --- .travis.yml | 4 - requirements/base.txt | 7 +- samcli/__init__.py | 2 +- samcli/cli/options.py | 1 + samcli/commands/_utils/options.py | 3 +- samcli/commands/_utils/template.py | 193 ++++++++++++- samcli/commands/build/build_context.py | 4 + samcli/commands/build/command.py | 10 +- .../local/cli_common/invoke_context.py | 39 ++- samcli/commands/local/cli_common/options.py | 46 ++- .../local/cli_common/user_exceptions.py | 35 +++ samcli/commands/local/invoke/cli.py | 28 +- samcli/commands/local/lib/exceptions.py | 9 + samcli/commands/local/lib/local_lambda.py | 49 +--- samcli/commands/local/lib/provider.py | 145 +++++++++- .../local/lib/sam_function_provider.py | 139 +++++++-- samcli/commands/local/start_api/cli.py | 29 +- samcli/commands/local/start_lambda/cli.py | 30 +- .../validate/lib/sam_template_validator.py | 4 + samcli/lib/build/app_builder.py | 10 +- samcli/lib/utils/codeuri.py | 46 +++ samcli/lib/utils/progressbar.py | 25 ++ samcli/lib/utils/tar.py | 37 +++ samcli/local/docker/lambda_container.py | 88 +++--- samcli/local/docker/lambda_image.py | 226 +++++++++++++++ samcli/local/docker/manager.py | 8 +- samcli/local/init/__init__.py | 2 + .../cookiecutter.json | 2 +- .../{{cookiecutter.project_name}}/README.md | 20 +- .../template.yaml | 4 +- .../.gitignore | 168 +++++++++++ .../cookiecutter-aws-sam-hello-ruby/LICENSE | 14 + .../cookiecutter-aws-sam-hello-ruby/README.md | 62 ++++ .../cookiecutter.json | 4 + .../tests/test_cookiecutter.py | 39 +++ .../{{cookiecutter.project_name}}/.gitignore | 244 ++++++++++++++++ .../{{cookiecutter.project_name}}/Gemfile | 3 + .../{{cookiecutter.project_name}}/README.md | 169 +++++++++++ .../hello_world/app.rb | 83 ++++++ .../template.yaml | 44 +++ .../tests/unit/test_handler.rb | 91 ++++++ samcli/local/lambdafn/config.py | 28 +- samcli/local/lambdafn/runtime.py | 13 +- samcli/local/lambdafn/zip.py | 74 ++++- samcli/local/layers/__init__.py | 0 samcli/local/layers/layer_downloader.py | 177 ++++++++++++ .../local/lib/test_local_api_service.py | 10 +- .../commands/local/lib/test_local_lambda.py | 10 +- .../lib/test_sam_template_validator.py | 22 ++ .../local/apigw/test_local_apigw_service.py | 26 +- .../local/docker/test_lambda_container.py | 21 +- .../test_local_lambda_invoke.py | 15 +- .../functional/local/lambdafn/test_runtime.py | 15 +- tests/integration/buildcmd/test_build_cmd.py | 8 + .../local/invoke/invoke_integ_base.py | 26 +- tests/integration/local/invoke/layer_utils.py | 49 ++++ .../invoke/runtimes/test_with_runtime_zips.py | 24 +- .../local/invoke/test_integrations_cli.py | 213 ++++++++++++++ tests/integration/testdata/__init__.py | 0 .../testdata/buildcmd/template.yaml | 5 +- tests/integration/testdata/invoke/__init__.py | 0 .../invoke/layer_zips/changedlayer1.zip | Bin 0 -> 576 bytes .../testdata/invoke/layer_zips/layer1.zip | Bin 0 -> 1963 bytes .../testdata/invoke/layer_zips/layer2.zip | Bin 0 -> 1963 bytes .../testdata/invoke/layers/__init__.py | 0 .../invoke/layers/custom_layer/__init__.py | 0 .../layers/custom_layer/my_layer/__init__.py | 0 .../custom_layer/my_layer/simple_python.py | 2 + .../testdata/invoke/layers/layer-main.py | 21 ++ .../testdata/invoke/layers/layer-template.yml | 146 ++++++++++ .../invoke/runtimes/custom_bash/bootstrap | 32 +++ .../invoke/runtimes/custom_bash/hello.sh | 7 + .../testdata/invoke/runtimes/template.yaml | 14 + .../integration/testdata/invoke/template.yml | 2 +- tests/unit/commands/_utils/test_template.py | 141 ++++++++- tests/unit/commands/buildcmd/test_command.py | 26 +- .../local/cli_common/test_invoke_context.py | 36 ++- tests/unit/commands/local/invoke/test_cli.py | 103 ++++--- .../commands/local/lib/test_local_lambda.py | 164 +++-------- .../unit/commands/local/lib/test_provider.py | 58 ++++ .../local/lib/test_sam_function_provider.py | 135 +++++++-- .../unit/commands/local/start_api/test_cli.py | 42 ++- .../commands/local/start_lambda/test_cli.py | 42 ++- .../lib/test_sam_template_validator.py | 14 +- .../unit/lib/build_module/test_app_builder.py | 4 +- tests/unit/lib/utils/test_codeuri.py | 68 +++++ tests/unit/lib/utils/test_progressbar.py | 18 ++ tests/unit/lib/utils/test_tar.py | 28 ++ .../local/docker/test_lambda_container.py | 26 +- tests/unit/local/docker/test_lambda_image.py | 269 ++++++++++++++++++ tests/unit/local/docker/test_manager.py | 19 ++ tests/unit/local/lambdafn/test_config.py | 7 +- tests/unit/local/lambdafn/test_runtime.py | 23 +- tests/unit/local/lambdafn/test_zip.py | 99 ++++++- tests/unit/local/layers/__init__.py | 0 .../unit/local/layers/test_download_layers.py | 197 +++++++++++++ 96 files changed, 4147 insertions(+), 498 deletions(-) create mode 100644 samcli/lib/utils/codeuri.py create mode 100644 samcli/lib/utils/progressbar.py create mode 100644 samcli/lib/utils/tar.py create mode 100644 samcli/local/docker/lambda_image.py create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/.gitignore create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/LICENSE create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/README.md create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/cookiecutter.json create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/tests/test_cookiecutter.py create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/.gitignore create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/Gemfile create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/README.md create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/hello_world/app.rb create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/template.yaml create mode 100644 samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/tests/unit/test_handler.rb create mode 100644 samcli/local/layers/__init__.py create mode 100644 samcli/local/layers/layer_downloader.py create mode 100644 tests/integration/local/invoke/layer_utils.py create mode 100644 tests/integration/testdata/__init__.py create mode 100644 tests/integration/testdata/invoke/__init__.py create mode 100644 tests/integration/testdata/invoke/layer_zips/changedlayer1.zip create mode 100644 tests/integration/testdata/invoke/layer_zips/layer1.zip create mode 100644 tests/integration/testdata/invoke/layer_zips/layer2.zip create mode 100644 tests/integration/testdata/invoke/layers/__init__.py create mode 100644 tests/integration/testdata/invoke/layers/custom_layer/__init__.py create mode 100644 tests/integration/testdata/invoke/layers/custom_layer/my_layer/__init__.py create mode 100644 tests/integration/testdata/invoke/layers/custom_layer/my_layer/simple_python.py create mode 100644 tests/integration/testdata/invoke/layers/layer-main.py create mode 100644 tests/integration/testdata/invoke/layers/layer-template.yml create mode 100755 tests/integration/testdata/invoke/runtimes/custom_bash/bootstrap create mode 100755 tests/integration/testdata/invoke/runtimes/custom_bash/hello.sh create mode 100644 tests/unit/commands/local/lib/test_provider.py create mode 100644 tests/unit/lib/utils/test_codeuri.py create mode 100644 tests/unit/lib/utils/test_progressbar.py create mode 100644 tests/unit/lib/utils/test_tar.py create mode 100644 tests/unit/local/docker/test_lambda_image.py create mode 100644 tests/unit/local/layers/__init__.py create mode 100644 tests/unit/local/layers/test_download_layers.py diff --git a/.travis.yml b/.travis.yml index dfb0b9382f..23de2429a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,6 @@ python: - "2.7" - "3.6" -env: - global: - - AWS_DEFAULT_REGION=us-east-1 - # Enable 3.7 without globally enabling sudo and dist: xenial for other build jobs matrix: include: diff --git a/requirements/base.txt b/requirements/base.txt index 4fe8481b6f..6d50131d4f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,12 +3,13 @@ chevron~=0.12 click~=6.7 enum34~=1.1.6; python_version<"3.4" Flask~=1.0.2 -boto3~=1.5 +boto3~=1.9, >=1.9.56 PyYAML~=3.12 cookiecutter~=1.6.0 -aws-sam-translator==1.8.0 +aws-sam-translator==1.9.0 docker>=3.3.0 dateparser~=0.7 python-dateutil~=2.6 pathlib2~=2.3.2; python_version<"3.4" -aws_lambda_builders==0.0.2 +requests~=2.20.0 +aws_lambda_builders==0.0.3 diff --git a/samcli/__init__.py b/samcli/__init__.py index 833faf0af2..a289e93c5e 100644 --- a/samcli/__init__.py +++ b/samcli/__init__.py @@ -2,4 +2,4 @@ SAM CLI version """ -__version__ = '0.7.0' +__version__ = '0.8.0' diff --git a/samcli/cli/options.py b/samcli/cli/options.py index c5e7baaff5..153706a1e3 100644 --- a/samcli/cli/options.py +++ b/samcli/cli/options.py @@ -41,6 +41,7 @@ def callback(ctx, param, value): return click.option('--region', expose_value=False, help='Set the AWS Region of the service (e.g. us-east-1).', + default='us-east-1', callback=callback)(f) diff --git a/samcli/commands/_utils/options.py b/samcli/commands/_utils/options.py index 270aac92bd..bea74343d7 100644 --- a/samcli/commands/_utils/options.py +++ b/samcli/commands/_utils/options.py @@ -96,7 +96,8 @@ def docker_click_options(): click.option('--skip-pull-image', is_flag=True, help="Specify whether CLI should skip pulling down the latest Docker image for Lambda runtime.", - envvar="SAM_SKIP_PULL_IMAGE"), + envvar="SAM_SKIP_PULL_IMAGE", + default=False), click.option('--docker-network', envvar="SAM_DOCKER_NETWORK", diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py index f44e285ef0..d0602c5305 100644 --- a/samcli/commands/_utils/template.py +++ b/samcli/commands/_utils/template.py @@ -2,6 +2,8 @@ Utilities to manipulate template """ +import os +import six import yaml try: @@ -9,15 +11,36 @@ except ImportError: import pathlib2 as pathlib -from samcli.yamlhelper import yaml_parse +from samcli.yamlhelper import yaml_parse, yaml_dump + + +_RESOURCES_WITH_LOCAL_PATHS = { + "AWS::Serverless::Function": ["CodeUri"], + "AWS::Serverless::Api": ["DefinitionUri"], + "AWS::AppSync::GraphQLSchema": ["DefinitionS3Location"], + "AWS::AppSync::Resolver": ["RequestMappingTemplateS3Location", "ResponseMappingTemplateS3Location"], + "AWS::Lambda::Function": ["Code"], + "AWS::ApiGateway::RestApi": ["BodyS3Location"], + "AWS::ElasticBeanstalk::ApplicationVersion": ["SourceBundle"], + "AWS::CloudFormation::Stack": ["TemplateURL"], + "AWS::Serverless::Application": ["Location"], + "AWS::Lambda::LayerVersion": ["Content"], + "AWS::Serverless::LayerVersion": ["ContentUri"] +} def get_template_data(template_file): """ Read the template file, parse it as JSON/YAML and return the template as a dictionary. - :param string template_file: Path to the template to read - :return dict: Template data as a dictionary + Parameters + ---------- + template_file : string + Path to the template to read + + Returns + ------- + Template data as a dictionary """ if not pathlib.Path(template_file).exists(): @@ -28,3 +51,167 @@ def get_template_data(template_file): return yaml_parse(fp.read()) except (ValueError, yaml.YAMLError) as ex: raise ValueError("Failed to parse template: {}".format(str(ex))) + + +def move_template(src_template_path, + dest_template_path, + template_dict): + """ + Move the SAM/CloudFormation template from ``src_template_path`` to ``dest_template_path``. For convenience, this + method accepts a dictionary of template data ``template_dict`` that will be written to the destination instead of + reading from the source file. + + SAM/CloudFormation template can contain certain properties whose value is a relative path to a local file/folder. + This path is always relative to the template's location. Before writing the template to ``dest_template_path`, + we will update these paths to be relative to the new location. + + This methods updates resource properties supported by ``aws cloudformation package`` command: + https://docs.aws.amazon.com/cli/latest/reference/cloudformation/package.html + + You must use this method if you are reading a template from one location, modifying it, and writing it back to a + different location. + + Parameters + ---------- + src_template_path : str + Path to the original location of the template + + dest_template_path : str + Path to the destination location where updated template should be written to + + template_dict : dict + Dictionary containing template contents. This dictionary will be updated & written to ``dest`` location. + """ + + original_root = os.path.dirname(src_template_path) + new_root = os.path.dirname(dest_template_path) + + # Next up, we will be writing the template to a different location. Before doing so, we should + # update any relative paths in the template to be relative to the new location. + modified_template = _update_relative_paths(template_dict, + original_root, + new_root) + + with open(dest_template_path, "w") as fp: + fp.write(yaml_dump(modified_template)) + + +def _update_relative_paths(template_dict, + original_root, + new_root): + """ + SAM/CloudFormation template can contain certain properties whose value is a relative path to a local file/folder. + This path is usually relative to the template's location. If the template is being moved from original location + ``original_root`` to new location ``new_root``, use this method to update these paths to be + relative to ``new_root``. + + After this method is complete, it is safe to write the template to ``new_root`` without + breaking any relative paths. + + This methods updates resource properties supported by ``aws cloudformation package`` command: + https://docs.aws.amazon.com/cli/latest/reference/cloudformation/package.html + + If a property is either an absolute path or a S3 URI, this method will not update them. + + + Parameters + ---------- + template_dict : dict + Dictionary containing template contents. This dictionary will be updated & written to ``dest`` location. + + original_root : str + Path to the directory where all paths were originally set relative to. This is usually the directory + containing the template originally + + new_root : str + Path to the new directory that all paths set relative to after this method completes. + + Returns + ------- + Updated dictionary + + """ + + for _, resource in template_dict.get("Resources", {}).items(): + resource_type = resource.get("Type") + + if resource_type not in _RESOURCES_WITH_LOCAL_PATHS: + # Unknown resource. Skipping + continue + + for path_prop_name in _RESOURCES_WITH_LOCAL_PATHS[resource_type]: + properties = resource.get("Properties", {}) + path = properties.get(path_prop_name) + + updated_path = _resolve_relative_to(path, original_root, new_root) + if not updated_path: + # This path does not need to get updated + continue + + properties[path_prop_name] = updated_path + + # AWS::Includes can be anywhere within the template dictionary. Hence we need to recurse through the + # dictionary in a separate method to find and update relative paths in there + template_dict = _update_aws_include_relative_path(template_dict, original_root, new_root) + + return template_dict + + +def _update_aws_include_relative_path(template_dict, original_root, new_root): + """ + Update relative paths in "AWS::Include" directive. This directive can be present at any part of the template, + and not just within resources. + """ + + for key, val in template_dict.items(): + if key == "Fn::Transform": + if isinstance(val, dict) and val.get("Name") == "AWS::Include": + path = val.get("Parameters", {}).get("Location", {}) + updated_path = _resolve_relative_to(path, original_root, new_root) + if not updated_path: + # This path does not need to get updated + continue + + val["Parameters"]["Location"] = updated_path + + # Recurse through all dictionary values + elif isinstance(val, dict): + _update_aws_include_relative_path(val, original_root, new_root) + elif isinstance(val, list): + for item in val: + if isinstance(item, dict): + _update_aws_include_relative_path(item, original_root, new_root) + + return template_dict + + +def _resolve_relative_to(path, original_root, new_root): + """ + If the given ``path`` is a relative path, then assume it is relative to ``original_root``. This method will + update the path to be resolve it relative to ``new_root`` and return. + + Examples + ------- + # Assume a file called template.txt at location /tmp/original/root/template.txt expressed as relative path + # We are trying to update it to be relative to /tmp/new/root instead of the /tmp/original/root + >>> result = _resolve_relative_to("template.txt", \ + "/tmp/original/root", \ + "/tmp/new/root") + >>> result + ../../original/root/template.txt + + Returns + ------- + Updated path if the given path is a relative path. None, if the path is not a relative path. + """ + + if not isinstance(path, six.string_types) \ + or path.startswith("s3://") \ + or os.path.isabs(path): + # Value is definitely NOT a relative path. It is either a S3 URi or Absolute path or not a string at all + return None + + # Value is definitely a relative path. Change it relative to the destination directory + return os.path.relpath( + os.path.normpath(os.path.join(original_root, path)), # Absolute original path w.r.t ``original_root`` + new_root) # Resolve the original path with respect to ``new_root`` diff --git a/samcli/commands/build/build_context.py b/samcli/commands/build/build_context.py index 635fa128c9..5c4dd9f947 100644 --- a/samcli/commands/build/build_context.py +++ b/samcli/commands/build/build_context.py @@ -117,6 +117,10 @@ def use_container(self): def output_template_path(self): return os.path.join(self._build_dir, "template.yaml") + @property + def original_template_path(self): + return os.path.abspath(self._template_file) + @property def manifest_path_override(self): if self._manifest_path: diff --git a/samcli/commands/build/command.py b/samcli/commands/build/command.py index eb9eb80bd2..0911732627 100644 --- a/samcli/commands/build/command.py +++ b/samcli/commands/build/command.py @@ -7,13 +7,13 @@ import click from samcli.commands.exceptions import UserException -from samcli.yamlhelper import yaml_dump from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options from samcli.commands._utils.options import template_option_without_build, docker_common_options, \ parameter_override_option from samcli.commands.build.build_context import BuildContext from samcli.lib.build.app_builder import ApplicationBuilder, UnsupportedRuntimeException, \ BuildError, UnsupportedBuilderLibraryVersionError +from samcli.commands._utils.template import move_template LOG = logging.getLogger(__name__) @@ -32,6 +32,7 @@ ------------------ 1. Python2.7\n 2. Python3.6\n +3. Python3.7\n \b Examples -------- @@ -130,11 +131,12 @@ def do_cli(template, # pylint: disable=too-many-locals try: artifacts = builder.build() modified_template = builder.update_template(ctx.template_dict, - ctx.output_template_path, + ctx.original_template_path, artifacts) - with open(ctx.output_template_path, "w") as fp: - fp.write(yaml_dump(modified_template)) + move_template(ctx.original_template_path, + ctx.output_template_path, + modified_template) click.secho("\nBuild Succeeded", fg="green") diff --git a/samcli/commands/local/cli_common/invoke_context.py b/samcli/commands/local/cli_common/invoke_context.py index d003e154bc..caa5a06d87 100644 --- a/samcli/commands/local/cli_common/invoke_context.py +++ b/samcli/commands/local/cli_common/invoke_context.py @@ -13,8 +13,10 @@ from samcli.commands.local.lib.local_lambda import LocalLambdaRunner from samcli.commands.local.lib.debug_context import DebugContext from samcli.local.lambdafn.runtime import LambdaRuntime +from samcli.local.docker.lambda_image import LambdaImage from samcli.local.docker.manager import ContainerManager from samcli.commands._utils.template import get_template_data +from samcli.local.layers.layer_downloader import LayerDownloader from .user_exceptions import InvokeContextException, DebugContextException from ..lib.sam_function_provider import SamFunctionProvider @@ -43,7 +45,7 @@ class InvokeContext(object): This class sets up some resources that need to be cleaned up after the context object is used. """ - def __init__(self, + def __init__(self, # pylint: disable=R0914 template_file, function_identifier=None, env_vars_file=None, @@ -51,12 +53,13 @@ def __init__(self, docker_network=None, log_file=None, skip_pull_image=None, - aws_profile=None, debug_port=None, debug_args=None, debugger_path=None, - aws_region=None, - parameter_overrides=None): + parameter_overrides=None, + layer_cache_basedir=None, + force_image_build=None, + aws_region=None): """ Initialize the context @@ -85,13 +88,14 @@ def __init__(self, Additional arguments passed to the debugger debugger_path str Path to the directory of the debugger to mount on Docker - aws_profile str - AWS Credential profile to use - aws_region str - AWS region to use parameter_overrides dict Values for the template parameters - + layer_cache_basedir str + String representing the path to the layer cache directory + force_image_build bool + Whether or not to force build the image + aws_region str + AWS region to use """ self._template_file = template_file self._function_identifier = function_identifier @@ -100,18 +104,20 @@ def __init__(self, self._docker_network = docker_network self._log_file = log_file self._skip_pull_image = skip_pull_image - self._aws_profile = aws_profile - self._aws_region = aws_region self._debug_port = debug_port self._debug_args = debug_args self._debugger_path = debugger_path self._parameter_overrides = parameter_overrides or {} + self._layer_cache_basedir = layer_cache_basedir + self._force_image_build = force_image_build + self._aws_region = aws_region self._template_dict = None self._function_provider = None self._env_vars_value = None self._log_file_handle = None self._debug_context = None + self._layers_downloader = None def __enter__(self): """ @@ -182,14 +188,17 @@ def local_lambda_runner(self): container_manager = ContainerManager(docker_network_id=self._docker_network, skip_pull_image=self._skip_pull_image) - lambda_runtime = LambdaRuntime(container_manager) + layer_downloader = LayerDownloader(self._layer_cache_basedir, self.get_cwd()) + image_builder = LambdaImage(layer_downloader, + self._skip_pull_image, + self._force_image_build) + + lambda_runtime = LambdaRuntime(container_manager, image_builder) return LocalLambdaRunner(local_runtime=lambda_runtime, function_provider=self._function_provider, cwd=self.get_cwd(), env_vars_values=self._env_vars_value, - debug_context=self._debug_context, - aws_profile=self._aws_profile, - aws_region=self._aws_region) + debug_context=self._debug_context) @property def stdout(self): diff --git a/samcli/commands/local/cli_common/options.py b/samcli/commands/local/cli_common/options.py index 0d1c559f24..f8ad498f9b 100644 --- a/samcli/commands/local/cli_common/options.py +++ b/samcli/commands/local/cli_common/options.py @@ -5,6 +5,36 @@ import click from samcli.commands._utils.options import template_click_option, docker_click_options, parameter_override_click_option +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +def get_application_dir(): + """ + + Returns + ------- + Path + Path representing the application config directory + """ + return Path(click.get_app_dir('AWS SAM', force_posix=True)) + + +def get_default_layer_cache_dir(): + """ + Default the layer cache directory + + Returns + ------- + str + String representing the layer cache directory + """ + layer_cache_dir = get_application_dir().joinpath('layers-pkg') + + return str(layer_cache_dir) + def service_common_options(port): def construct_options(f): @@ -77,13 +107,19 @@ def invoke_common_options(f): click.option('--log-file', '-l', help="logfile to send runtime logs to."), - ] + docker_click_options() + [ + click.option('--layer-cache-basedir', + type=click.Path(exists=False, file_okay=False), + envvar="SAM_LAYER_CACHE_BASEDIR", + help="Specifies the location basedir where the Layers your template uses will be downloaded to.", + default=get_default_layer_cache_dir()), - click.option('--profile', - help="Specify which AWS credentials profile to use."), + ] + docker_click_options() + [ - click.option('--region', - help="Specify which AWS region to use."), + click.option('--force-image-build', + is_flag=True, + help='Specify whether CLI should rebuild the image used for invoking functions with layers.', + envvar='SAM_FORCE_IMAGE_BUILD', + default=False), ] diff --git a/samcli/commands/local/cli_common/user_exceptions.py b/samcli/commands/local/cli_common/user_exceptions.py index b2b739856c..74be2baf48 100644 --- a/samcli/commands/local/cli_common/user_exceptions.py +++ b/samcli/commands/local/cli_common/user_exceptions.py @@ -31,3 +31,38 @@ class DebugContextException(UserException): Something went wrong when creating the DebugContext """ pass + + +class ImageBuildException(UserException): + """ + Image failed to build + """ + pass + + +class CredentialsRequired(UserException): + """ + Credentials were not given when Required + """ + pass + + +class ResourceNotFound(UserException): + """ + The Resource requested was not found + """ + pass + + +class InvalidLayerVersionArn(UserException): + """ + The LayerVersion Arn given in the template is Invalid + """ + pass + + +class UnsupportedIntrinsic(UserException): + """ + Value from a template has an Intrinsic that is unsupported + """ + pass diff --git a/samcli/commands/local/invoke/cli.py b/samcli/commands/local/invoke/cli.py index 9e7a85b0b7..071a5d5143 100644 --- a/samcli/commands/local/invoke/cli.py +++ b/samcli/commands/local/invoke/cli.py @@ -5,19 +5,20 @@ import logging import click -from samcli.cli.main import pass_context, common_options as cli_framework_options +from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options from samcli.commands.local.cli_common.options import invoke_common_options from samcli.commands.exceptions import UserException +from samcli.commands.local.lib.exceptions import InvalidLayerReference from samcli.commands.local.cli_common.invoke_context import InvokeContext from samcli.local.lambdafn.exceptions import FunctionNotFound from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.local.lib.exceptions import OverridesNotWellDefinedError from samcli.local.docker.manager import DockerImagePullFailedException +from samcli.local.docker.lambda_container import DebuggingNotSupported LOG = logging.getLogger(__name__) - HELP_TEXT = """ You can use this command to execute your function in a Lambda-like environment locally. You can pass in the event body via stdin or by using the -e (--event) parameter. @@ -41,22 +42,23 @@ @click.option("--no-event", is_flag=True, default=False, help="Invoke Function with an empty event") @invoke_common_options @cli_framework_options +@aws_creds_options @click.argument('function_identifier', required=False) @pass_context # pylint: disable=R0914 -def cli(ctx, function_identifier, template, event, no_event, env_vars, debug_port, - debug_args, debugger_path, docker_volume_basedir, docker_network, log_file, skip_pull_image, profile, region, +def cli(ctx, function_identifier, template, event, no_event, env_vars, debug_port, debug_args, debugger_path, + docker_volume_basedir, docker_network, log_file, layer_cache_basedir, skip_pull_image, force_image_build, parameter_overrides): # All logic must be implemented in the ``do_cli`` method. This helps with easy unit testing do_cli(ctx, function_identifier, template, event, no_event, env_vars, debug_port, debug_args, debugger_path, - docker_volume_basedir, docker_network, log_file, skip_pull_image, profile, region, + docker_volume_basedir, docker_network, log_file, layer_cache_basedir, skip_pull_image, force_image_build, parameter_overrides) # pragma: no cover def do_cli(ctx, function_identifier, template, event, no_event, env_vars, debug_port, # pylint: disable=R0914 - debug_args, debugger_path, docker_volume_basedir, docker_network, log_file, skip_pull_image, profile, - region, parameter_overrides): + debug_args, debugger_path, docker_volume_basedir, docker_network, log_file, layer_cache_basedir, + skip_pull_image, force_image_build, parameter_overrides): """ Implementation of the ``cli`` method, just separated out for unit testing purposes """ @@ -82,12 +84,13 @@ def do_cli(ctx, function_identifier, template, event, no_event, env_vars, debug_ docker_network=docker_network, log_file=log_file, skip_pull_image=skip_pull_image, - aws_profile=profile, debug_port=debug_port, debug_args=debug_args, debugger_path=debugger_path, - aws_region=region, - parameter_overrides=parameter_overrides) as context: + parameter_overrides=parameter_overrides, + layer_cache_basedir=layer_cache_basedir, + force_image_build=force_image_build, + aws_region=ctx.region) as context: # Invoke the function context.local_lambda_runner.invoke(context.function_name, @@ -97,7 +100,10 @@ def do_cli(ctx, function_identifier, template, event, no_event, env_vars, debug_ except FunctionNotFound: raise UserException("Function {} not found in template".format(function_identifier)) - except (InvalidSamDocumentException, OverridesNotWellDefinedError) as ex: + except (InvalidSamDocumentException, + OverridesNotWellDefinedError, + InvalidLayerReference, + DebuggingNotSupported) as ex: raise UserException(str(ex)) except DockerImagePullFailedException as ex: raise UserException(str(ex)) diff --git a/samcli/commands/local/lib/exceptions.py b/samcli/commands/local/lib/exceptions.py index 1df90f9217..32e8afe91d 100644 --- a/samcli/commands/local/lib/exceptions.py +++ b/samcli/commands/local/lib/exceptions.py @@ -15,3 +15,12 @@ class OverridesNotWellDefinedError(Exception): Raised when the overrides file is invalid """ pass + + +class InvalidLayerReference(Exception): + """ + Raised when the LayerVersion LogicalId does not exist in the template + """ + def __init__(self): + super(InvalidLayerReference, self).__init__("Layer References need to be of type " + "'AWS::Serverless::LayerVersion' or 'AWS::Lambda::LayerVersion'") diff --git a/samcli/commands/local/lib/local_lambda.py b/samcli/commands/local/lib/local_lambda.py index 12455b2788..6a3e00f4c7 100644 --- a/samcli/commands/local/lib/local_lambda.py +++ b/samcli/commands/local/lib/local_lambda.py @@ -6,6 +6,7 @@ import logging import boto3 +from samcli.lib.utils.codeuri import resolve_code_path from samcli.local.lambdafn.env_vars import EnvironmentVariables from samcli.local.lambdafn.config import FunctionConfig from samcli.local.lambdafn.exceptions import FunctionNotFound @@ -19,7 +20,6 @@ class LocalLambdaRunner(object): Runs Lambda functions locally. This class is a wrapper around the `samcli.local` library which takes care of actually running the function on a Docker container. """ - PRESENT_DIR = "." MAX_DEBUG_TIMEOUT = 36000 # 10 hours in seconds def __init__(self, @@ -27,9 +27,7 @@ def __init__(self, function_provider, cwd, env_vars_values=None, - aws_profile=None, - debug_context=None, - aws_region=None): + debug_context=None): """ Initializes the class @@ -40,16 +38,12 @@ def __init__(self, :param dict env_vars_values: Optional. Dictionary containing values of environment variables :param integer debug_port: Optional. Port to bind the debugger to :param string debug_args: Optional. Additional arguments passed to the debugger - :param string aws_profile: Optional. AWS Credentials profile to use - :param string aws_region: Optional. AWS region to use """ self.local_runtime = local_runtime self.provider = function_provider self.cwd = cwd self.env_vars_values = env_vars_values or {} - self.aws_profile = aws_profile - self.aws_region = aws_region self.debug_context = debug_context def invoke(self, function_name, event, stdout=None, stderr=None): @@ -101,7 +95,7 @@ def _get_invoke_config(self, function): """ env_vars = self._make_env_vars(function) - code_abs_path = self._get_code_path(function.codeuri) + code_abs_path = resolve_code_path(self.cwd, function.codeuri) LOG.debug("Resolved absolute path to code is %s", code_abs_path) @@ -117,6 +111,7 @@ def _get_invoke_config(self, function): runtime=function.runtime, handler=function.handler, code_abs_path=code_abs_path, + layers=function.layers, memory=function.memory, timeout=function_timeout, env_vars=env_vars) @@ -183,32 +178,6 @@ def _make_env_vars(self, function): override_values=overrides, aws_creds=aws_creds) - def _get_code_path(self, codeuri): - """ - Returns path to the function code resolved based on current working directory. - - :param string codeuri: CodeURI of the function. This should contain the path to the function code - :return string: Absolute path to the function code - """ - - LOG.debug("Resolving code path. Cwd=%s, CodeUri=%s", self.cwd, codeuri) - - # First, let us figure out the current working directory. - # If current working directory is not provided, then default to the directory where the CLI is running from - if not self.cwd or self.cwd == self.PRESENT_DIR: - self.cwd = os.getcwd() - - # Make sure cwd is an absolute path - self.cwd = os.path.abspath(self.cwd) - - # Next, let us get absolute path of function code. - # Codepath is always relative to current working directory - # If the path is relative, then construct the absolute version - if not os.path.isabs(codeuri): - codeuri = os.path.normpath(os.path.join(self.cwd, codeuri)) - - return codeuri - def get_aws_creds(self): """ Returns AWS credentials obtained from the shell environment or given profile @@ -219,10 +188,14 @@ def get_aws_creds(self): """ result = {} - LOG.debug("Loading AWS credentials from session with profile '%s'", self.aws_profile) - # TODO: Consider changing it to use boto3 default session. We already have an annotation # to pass command line arguments for region & profile to setup boto3 default session - session = boto3.session.Session(profile_name=self.aws_profile, region_name=self.aws_region) + if boto3.DEFAULT_SESSION: + session = boto3.DEFAULT_SESSION + else: + session = boto3.session.Session() + + profile_name = session.profile_name if session else None + LOG.debug("Loading AWS credentials from session with profile '%s'", profile_name) if not session: return result diff --git a/samcli/commands/local/lib/provider.py b/samcli/commands/local/lib/provider.py index 7e3183fc7b..c8265c81e6 100644 --- a/samcli/commands/local/lib/provider.py +++ b/samcli/commands/local/lib/provider.py @@ -2,9 +2,13 @@ A provider class that can parse and return Lambda Functions from a variety of sources. A SAM template is one such source """ - +import hashlib from collections import namedtuple +import six + +from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn, UnsupportedIntrinsic + # Named Tuple to representing the properties of a Lambda Function Function = namedtuple("Function", [ # Function name or logical ID @@ -31,10 +35,147 @@ # Lambda Execution IAM Role ARN. In the future, this can be used by Local Lambda runtime to assume the IAM role # to get credentials to run the container with. This gives a much higher fidelity simulation of cloud Lambda. - "rolearn" + "rolearn", + + # List of Layers + "layers" ]) +class LayerVersion(object): + """ + Represents the LayerVersion Resource for AWS Lambda + """ + + LAYER_NAME_DELIMETER = "-" + + def __init__(self, arn, codeuri): + """ + + Parameters + ---------- + name str + Name of the layer, this can be the ARN or Logical Id in the template + codeuri str + CodeURI of the layer. This should contain the path to the layer code + """ + if not isinstance(arn, six.string_types): + raise UnsupportedIntrinsic("{} is an Unsupported Intrinsic".format(arn)) + + self._arn = arn + self._codeuri = codeuri + self.is_defined_within_template = bool(codeuri) + self._name = LayerVersion._compute_layer_name(self.is_defined_within_template, arn) + self._version = LayerVersion._compute_layer_version(self.is_defined_within_template, arn) + + @staticmethod + def _compute_layer_version(is_defined_within_template, arn): + """ + Parses out the Layer version from the arn + + Parameters + ---------- + is_defined_within_template bool + True if the resource is a Ref to a resource otherwise False + arn str + ARN of the Resource + + Returns + ------- + int + The Version of the LayerVersion + + """ + + if is_defined_within_template: + return None + + try: + _, layer_version = arn.rsplit(':', 1) + layer_version = int(layer_version) + except ValueError: + raise InvalidLayerVersionArn(arn + " is an Invalid Layer Arn.") + + return layer_version + + @staticmethod + def _compute_layer_name(is_defined_within_template, arn): + """ + Computes a unique name based on the LayerVersion Arn + + Format: + -- + + Parameters + ---------- + is_defined_within_template bool + True if the resource is a Ref to a resource otherwise False + arn str + ARN of the Resource + + Returns + ------- + str + A unique name that represents the LayerVersion + """ + + # If the Layer is defined in the template, the arn will represent the LogicalId of the LayerVersion Resource, + # which does not require creating a name based on the arn. + if is_defined_within_template: + return arn + + try: + _, layer_name, layer_version = arn.rsplit(':', 2) + except ValueError: + raise InvalidLayerVersionArn(arn + " is an Invalid Layer Arn.") + + return LayerVersion.LAYER_NAME_DELIMETER.join([layer_name, + layer_version, + hashlib.sha256(arn.encode('utf-8')).hexdigest()[0:10]]) + + @property + def arn(self): + return self._arn + + @property + def name(self): + """ + A unique name from the arn or logical id of the Layer + + A LayerVersion Arn example: + arn:aws:lambda:region:account-id:layer:layer-name:version + + Returns + ------- + str + A name of the Layer that is used on the system to uniquely identify the layer + """ + return self._name + + @property + def codeuri(self): + return self._codeuri + + @property + def version(self): + return self._version + + @property + def layer_arn(self): + layer_arn, _ = self.arn.rsplit(':', 1) + return layer_arn + + @codeuri.setter + def codeuri(self, codeuri): + self._codeuri = codeuri + + def __eq__(self, other): + if isinstance(other, type(self)): + return self.__dict__ == other.__dict__ + + return False + + class FunctionProvider(object): """ Abstract base class of the function provider. diff --git a/samcli/commands/local/lib/sam_function_provider.py b/samcli/commands/local/lib/sam_function_provider.py index c9f2cc1667..784593c3ec 100644 --- a/samcli/commands/local/lib/sam_function_provider.py +++ b/samcli/commands/local/lib/sam_function_provider.py @@ -5,7 +5,8 @@ import logging import six -from .provider import FunctionProvider, Function +from .exceptions import InvalidLayerReference +from .provider import FunctionProvider, Function, LayerVersion from .sam_base_provider import SamBaseProvider LOG = logging.getLogger(__name__) @@ -21,6 +22,8 @@ class SamFunctionProvider(FunctionProvider): _SERVERLESS_FUNCTION = "AWS::Serverless::Function" _LAMBDA_FUNCTION = "AWS::Lambda::Function" + _SERVERLESS_LAYER = "AWS::Serverless::LayerVersion" + _LAMBDA_LAYER = "AWS::Lambda::LayerVersion" _DEFAULT_CODEURI = "." def __init__(self, template_dict, parameter_overrides=None): @@ -93,17 +96,19 @@ def _extract_functions(resources): resource_properties = resource.get("Properties", {}) if resource_type == SamFunctionProvider._SERVERLESS_FUNCTION: - result[name] = SamFunctionProvider._convert_sam_function_resource(name, resource_properties) + layers = SamFunctionProvider._parse_layer_info(resource_properties.get("Layers", []), resources) + result[name] = SamFunctionProvider._convert_sam_function_resource(name, resource_properties, layers) elif resource_type == SamFunctionProvider._LAMBDA_FUNCTION: - result[name] = SamFunctionProvider._convert_lambda_function_resource(name, resource_properties) + layers = SamFunctionProvider._parse_layer_info(resource_properties.get("Layers", []), resources) + result[name] = SamFunctionProvider._convert_lambda_function_resource(name, resource_properties, layers) # We don't care about other resource types. Just ignore them return result @staticmethod - def _convert_sam_function_resource(name, resource_properties): + def _convert_sam_function_resource(name, resource_properties, layers): """ Converts a AWS::Serverless::Function resource to a Function configuration usable by the provider. @@ -113,15 +118,7 @@ def _convert_sam_function_resource(name, resource_properties): :return samcli.commands.local.lib.provider.Function: Function configuration """ - codeuri = resource_properties.get("CodeUri", SamFunctionProvider._DEFAULT_CODEURI) - - # CodeUri can be a dictionary of S3 Bucket/Key or a S3 URI, neither of which are supported - if isinstance(codeuri, dict) or \ - (isinstance(codeuri, six.string_types) and codeuri.startswith("s3://")): - - codeuri = SamFunctionProvider._DEFAULT_CODEURI - LOG.warning("Lambda function '%s' has specified S3 location for CodeUri which is unsupported. " - "Using default value of '%s' instead", name, codeuri) + codeuri = SamFunctionProvider._extract_sam_function_codeuri(name, resource_properties, "CodeUri") LOG.debug("Found Serverless function with name='%s' and CodeUri='%s'", name, codeuri) @@ -133,11 +130,40 @@ def _convert_sam_function_resource(name, resource_properties): handler=resource_properties.get("Handler"), codeuri=codeuri, environment=resource_properties.get("Environment"), - rolearn=resource_properties.get("Role") + rolearn=resource_properties.get("Role"), + layers=layers ) @staticmethod - def _convert_lambda_function_resource(name, resource_properties): # pylint: disable=invalid-name + def _extract_sam_function_codeuri(name, resource_properties, code_property_key): + """ + Extracts the SAM Function CodeUri from the Resource Properties + + Parameters + ---------- + name str + LogicalId of the resource + resource_properties dict + Dictionary representing the Properties of the Resource + code_property_key str + Property Key of the code on the Resource + + Returns + ------- + str + Representing the local code path + """ + codeuri = resource_properties.get(code_property_key, SamFunctionProvider._DEFAULT_CODEURI) + # CodeUri can be a dictionary of S3 Bucket/Key or a S3 URI, neither of which are supported + if isinstance(codeuri, dict) or \ + (isinstance(codeuri, six.string_types) and codeuri.startswith("s3://")): + codeuri = SamFunctionProvider._DEFAULT_CODEURI + LOG.warning("Lambda function '%s' has specified S3 location for CodeUri which is unsupported. " + "Using default value of '%s' instead", name, codeuri) + return codeuri + + @staticmethod + def _convert_lambda_function_resource(name, resource_properties, layers): # pylint: disable=invalid-name """ Converts a AWS::Serverless::Function resource to a Function configuration usable by the provider. @@ -149,7 +175,7 @@ def _convert_lambda_function_resource(name, resource_properties): # pylint: dis # CodeUri is set to "." in order to get code locally from current directory. AWS::Lambda::Function's ``Code`` # property does not support specifying a local path - codeuri = SamFunctionProvider._DEFAULT_CODEURI + codeuri = SamFunctionProvider._extract_lambda_function_code(resource_properties, "Code") LOG.debug("Found Lambda function with name='%s' and CodeUri='%s'", name, codeuri) @@ -161,5 +187,84 @@ def _convert_lambda_function_resource(name, resource_properties): # pylint: dis handler=resource_properties.get("Handler"), codeuri=codeuri, environment=resource_properties.get("Environment"), - rolearn=resource_properties.get("Role") + rolearn=resource_properties.get("Role"), + layers=layers ) + + @staticmethod + def _extract_lambda_function_code(resource_properties, code_property_key): + """ + Extracts the Lambda Function Code from the Resource Properties + + Parameters + ---------- + resource_properties dict + Dictionary representing the Properties of the Resource + code_property_key str + Property Key of the code on the Resource + + Returns + ------- + str + Representing the local code path + """ + + codeuri = resource_properties.get(code_property_key, SamFunctionProvider._DEFAULT_CODEURI) + + if isinstance(codeuri, dict): + codeuri = SamFunctionProvider._DEFAULT_CODEURI + + return codeuri + + @staticmethod + def _parse_layer_info(list_of_layers, resources): + """ + Creates a list of Layer objects that are represented by the resources and the list of layers + + Parameters + ---------- + list_of_layers List(str) + List of layers that are defined within the Layers Property on a function + resources dict + The Resources dictionary defined in a template + + Returns + ------- + List(samcli.commands.local.lib.provider.Layer) + List of the Layer objects created from the template and layer list defined on the function. The order + of the layers does not change. + + I.E: list_of_layers = ["layer1", "layer2"] the return would be [Layer("layer1"), Layer("layer2")] + """ + layers = [] + for layer in list_of_layers: + # If the layer is a string, assume it is the arn + if isinstance(layer, six.string_types): + layers.append(LayerVersion(layer, None)) + continue + + # In the list of layers that is defined within a template, you can reference a LayerVersion resource. + # When running locally, we need to follow that Ref so we can extract the local path to the layer code. + if isinstance(layer, dict) and layer.get("Ref"): + layer_logical_id = layer.get("Ref") + layer_resource = resources.get(layer_logical_id) + if not layer_resource or \ + layer_resource.get("Type", "") not in (SamFunctionProvider._SERVERLESS_LAYER, + SamFunctionProvider._LAMBDA_LAYER): + raise InvalidLayerReference() + + layer_properties = layer_resource.get("Properties", {}) + resource_type = layer_resource.get("Type") + codeuri = None + + if resource_type == SamFunctionProvider._LAMBDA_LAYER: + codeuri = SamFunctionProvider._extract_lambda_function_code(layer_properties, "Content") + + if resource_type == SamFunctionProvider._SERVERLESS_LAYER: + codeuri = SamFunctionProvider._extract_sam_function_codeuri(layer_logical_id, + layer_properties, + "ContentUri") + + layers.append(LayerVersion(layer_logical_id, codeuri)) + + return layers diff --git a/samcli/commands/local/start_api/cli.py b/samcli/commands/local/start_api/cli.py index 61a411b6b3..1a24c34908 100644 --- a/samcli/commands/local/start_api/cli.py +++ b/samcli/commands/local/start_api/cli.py @@ -5,14 +5,15 @@ import logging import click -from samcli.cli.main import pass_context, common_options as cli_framework_options +from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options from samcli.commands.local.cli_common.options import invoke_common_options, service_common_options from samcli.commands.local.cli_common.invoke_context import InvokeContext -from samcli.commands.local.lib.exceptions import NoApisDefined +from samcli.commands.local.lib.exceptions import NoApisDefined, InvalidLayerReference from samcli.commands.exceptions import UserException from samcli.commands.local.lib.local_api_service import LocalApiService from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.local.lib.exceptions import OverridesNotWellDefinedError +from samcli.local.docker.lambda_container import DebuggingNotSupported LOG = logging.getLogger(__name__) @@ -39,7 +40,8 @@ help="Any static assets (e.g. CSS/Javascript/HTML) files located in this directory " "will be presented at /") @invoke_common_options -@cli_framework_options # pylint: disable=R0914 +@cli_framework_options +@aws_creds_options # pylint: disable=R0914 @pass_context def cli(ctx, # start-api Specific Options @@ -47,18 +49,17 @@ def cli(ctx, # Common Options for Lambda Invoke template, env_vars, debug_port, debug_args, debugger_path, docker_volume_basedir, - docker_network, log_file, skip_pull_image, profile, region, parameter_overrides - ): + docker_network, log_file, layer_cache_basedir, skip_pull_image, force_image_build, parameter_overrides): # All logic must be implemented in the ``do_cli`` method. This helps with easy unit testing do_cli(ctx, host, port, static_dir, template, env_vars, debug_port, debug_args, debugger_path, - docker_volume_basedir, docker_network, log_file, skip_pull_image, profile, region, + docker_volume_basedir, docker_network, log_file, layer_cache_basedir, skip_pull_image, force_image_build, parameter_overrides) # pragma: no cover def do_cli(ctx, host, port, static_dir, template, env_vars, debug_port, debug_args, # pylint: disable=R0914 - debugger_path, docker_volume_basedir, docker_network, log_file, skip_pull_image, profile, region, - parameter_overrides): + debugger_path, docker_volume_basedir, docker_network, log_file, layer_cache_basedir, skip_pull_image, + force_image_build, parameter_overrides): """ Implementation of the ``cli`` method, just separated out for unit testing purposes """ @@ -76,12 +77,13 @@ def do_cli(ctx, host, port, static_dir, template, env_vars, debug_port, debug_ar docker_network=docker_network, log_file=log_file, skip_pull_image=skip_pull_image, - aws_profile=profile, debug_port=debug_port, debug_args=debug_args, debugger_path=debugger_path, - aws_region=region, - parameter_overrides=parameter_overrides) as invoke_context: + parameter_overrides=parameter_overrides, + layer_cache_basedir=layer_cache_basedir, + force_image_build=force_image_build, + aws_region=ctx.region) as invoke_context: service = LocalApiService(lambda_invoke_context=invoke_context, port=port, @@ -91,5 +93,8 @@ def do_cli(ctx, host, port, static_dir, template, env_vars, debug_port, debug_ar except NoApisDefined: raise UserException("Template does not have any APIs connected to Lambda functions") - except (InvalidSamDocumentException, OverridesNotWellDefinedError) as ex: + except (InvalidSamDocumentException, + OverridesNotWellDefinedError, + InvalidLayerReference, + DebuggingNotSupported) as ex: raise UserException(str(ex)) diff --git a/samcli/commands/local/start_lambda/cli.py b/samcli/commands/local/start_lambda/cli.py index f84aadf573..6e7705d3b2 100644 --- a/samcli/commands/local/start_lambda/cli.py +++ b/samcli/commands/local/start_lambda/cli.py @@ -5,13 +5,15 @@ import logging import click -from samcli.cli.main import pass_context, common_options as cli_framework_options +from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options from samcli.commands.local.cli_common.options import invoke_common_options, service_common_options from samcli.commands.local.cli_common.invoke_context import InvokeContext from samcli.commands.local.cli_common.user_exceptions import UserException +from samcli.commands.local.lib.exceptions import InvalidLayerReference from samcli.commands.local.lib.local_lambda_service import LocalLambdaService from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.local.lib.exceptions import OverridesNotWellDefinedError +from samcli.local.docker.lambda_container import DebuggingNotSupported LOG = logging.getLogger(__name__) @@ -54,24 +56,26 @@ @service_common_options(3001) @invoke_common_options @cli_framework_options +@aws_creds_options @pass_context -def cli(ctx, +def cli(ctx, # pylint: disable=R0914 # start-lambda Specific Options host, port, # Common Options for Lambda Invoke template, env_vars, debug_port, debug_args, debugger_path, docker_volume_basedir, - docker_network, log_file, skip_pull_image, profile, region, parameter_overrides - ): + docker_network, log_file, layer_cache_basedir, skip_pull_image, force_image_build, + parameter_overrides): # pylint: disable=R0914 # All logic must be implemented in the ``do_cli`` method. This helps with easy unit testing do_cli(ctx, host, port, template, env_vars, debug_port, debug_args, debugger_path, docker_volume_basedir, - docker_network, log_file, skip_pull_image, profile, region, parameter_overrides) # pragma: no cover + docker_network, log_file, layer_cache_basedir, skip_pull_image, force_image_build, + parameter_overrides) # pragma: no cover def do_cli(ctx, host, port, template, env_vars, debug_port, debug_args, # pylint: disable=R0914 - debugger_path, docker_volume_basedir, docker_network, log_file, skip_pull_image, profile, region, - parameter_overrides): + debugger_path, docker_volume_basedir, docker_network, log_file, layer_cache_basedir, skip_pull_image, + force_image_build, parameter_overrides): """ Implementation of the ``cli`` method, just separated out for unit testing purposes """ @@ -89,17 +93,21 @@ def do_cli(ctx, host, port, template, env_vars, debug_port, debug_args, # pylin docker_network=docker_network, log_file=log_file, skip_pull_image=skip_pull_image, - aws_profile=profile, debug_port=debug_port, debug_args=debug_args, debugger_path=debugger_path, - aws_region=region, - parameter_overrides=parameter_overrides) as invoke_context: + parameter_overrides=parameter_overrides, + layer_cache_basedir=layer_cache_basedir, + force_image_build=force_image_build, + aws_region=ctx.region) as invoke_context: service = LocalLambdaService(lambda_invoke_context=invoke_context, port=port, host=host) service.start() - except (InvalidSamDocumentException, OverridesNotWellDefinedError) as ex: + except (InvalidSamDocumentException, + OverridesNotWellDefinedError, + InvalidLayerReference, + DebuggingNotSupported) as ex: raise UserException(str(ex)) diff --git a/samcli/commands/validate/lib/sam_template_validator.py b/samcli/commands/validate/lib/sam_template_validator.py index d73ce38195..57e46b56b9 100644 --- a/samcli/commands/validate/lib/sam_template_validator.py +++ b/samcli/commands/validate/lib/sam_template_validator.py @@ -104,6 +104,10 @@ def _replace_local_codeuri(self): SamTemplateValidator._update_to_s3_uri("CodeUri", resource_dict) + if resource_type == "AWS::Serverless::LayerVersion": + + SamTemplateValidator._update_to_s3_uri("ContentUri", resource_dict) + if resource_type == "AWS::Serverless::Api": if "DefinitionBody" not in resource_dict: SamTemplateValidator._update_to_s3_uri("DefinitionUri", resource_dict) diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index d097b0cded..f10453cf22 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -118,7 +118,7 @@ def build(self): return result - def update_template(self, template_dict, target_template_path, built_artifacts): + def update_template(self, template_dict, original_template_path, built_artifacts): """ Given the path to built artifacts, update the template to point appropriate resource CodeUris to the artifacts folder @@ -126,7 +126,7 @@ def update_template(self, template_dict, target_template_path, built_artifacts): Parameters ---------- template_dict - target_template_path : str + original_template_path : str Path where the template file will be written to built_artifacts : dict @@ -138,7 +138,7 @@ def update_template(self, template_dict, target_template_path, built_artifacts): Updated template """ - target_dir = os.path.dirname(target_template_path) + original_dir = os.path.dirname(original_template_path) for logical_id, resource in template_dict.get("Resources", {}).items(): @@ -146,10 +146,10 @@ def update_template(self, template_dict, target_template_path, built_artifacts): # this resource was not built. So skip it continue - # Artifacts are written relative to the output template because it makes the template portability + # Artifacts are written relative to the template because it makes the template portable # Ex: A CI/CD pipeline build stage could zip the output folder and pass to a # package stage running on a different machine - artifact_relative_path = os.path.relpath(built_artifacts[logical_id], target_dir) + artifact_relative_path = os.path.relpath(built_artifacts[logical_id], original_dir) resource_type = resource.get("Type") properties = resource.setdefault("Properties", {}) diff --git a/samcli/lib/utils/codeuri.py b/samcli/lib/utils/codeuri.py new file mode 100644 index 0000000000..d6a69d2158 --- /dev/null +++ b/samcli/lib/utils/codeuri.py @@ -0,0 +1,46 @@ +""" +Contains CodeUri Related methods +""" + +import os +import logging + +LOG = logging.getLogger(__name__) + +PRESENT_DIR = "." + + +def resolve_code_path(cwd, codeuri): + """ + Returns path to the function code resolved based on current working directory. + + Parameters + ---------- + cwd str + Current working directory + codeuri + CodeURI of the function. This should contain the path to the function code + + Returns + ------- + str + Absolute path to the function code + + """ + LOG.debug("Resolving code path. Cwd=%s, CodeUri=%s", cwd, codeuri) + + # First, let us figure out the current working directory. + # If current working directory is not provided, then default to the directory where the CLI is running from + if not cwd or cwd == PRESENT_DIR: + cwd = os.getcwd() + + # Make sure cwd is an absolute path + cwd = os.path.abspath(cwd) + + # Next, let us get absolute path of function code. + # Codepath is always relative to current working directory + # If the path is relative, then construct the absolute version + if not os.path.isabs(codeuri): + codeuri = os.path.normpath(os.path.join(cwd, codeuri)) + + return codeuri diff --git a/samcli/lib/utils/progressbar.py b/samcli/lib/utils/progressbar.py new file mode 100644 index 0000000000..6caf8cf44d --- /dev/null +++ b/samcli/lib/utils/progressbar.py @@ -0,0 +1,25 @@ +""" +ProgressBar operations +""" + +import click + + +def progressbar(length, label): + """ + Creates a progressbar + + Parameters + ---------- + length int + Length of the ProgressBar + label str + Label to give to the progressbar + + Returns + ------- + click.progressbar + Progressbar + + """ + return click.progressbar(length=length, label=label, show_pos=True) diff --git a/samcli/lib/utils/tar.py b/samcli/lib/utils/tar.py new file mode 100644 index 0000000000..b732faee47 --- /dev/null +++ b/samcli/lib/utils/tar.py @@ -0,0 +1,37 @@ +""" +Tarball Archive utility +""" + +import tarfile +from tempfile import TemporaryFile +from contextlib import contextmanager + + +@contextmanager +def create_tarball(tar_paths): + """ + Context Manger that creates the tarball of the Docker Context to use for building the image + + Parameters + ---------- + tar_paths dict(str, str) + Key representing a full path to the file or directory and the Value representing the path within the tarball + + Yields + ------ + The tarball file + """ + tarballfile = TemporaryFile() + + with tarfile.open(fileobj=tarballfile, mode='w:gz') as archive: + for path_on_system, path_in_tarball in tar_paths.items(): + archive.add(path_on_system, arcname=path_in_tarball) + + # Flush are seek to the beginning of the file + tarballfile.flush() + tarballfile.seek(0) + + try: + yield tarballfile + finally: + tarballfile.close() diff --git a/samcli/local/docker/lambda_container.py b/samcli/local/docker/lambda_container.py index 74c7ab51df..1a9c06a7da 100644 --- a/samcli/local/docker/lambda_container.py +++ b/samcli/local/docker/lambda_container.py @@ -1,32 +1,13 @@ """ Represents Lambda runtime containers. """ -from enum import Enum +import logging from .container import Container +from .lambda_image import Runtime -class Runtime(Enum): - nodejs = "nodejs" - nodejs43 = "nodejs4.3" - nodejs610 = "nodejs6.10" - nodejs810 = "nodejs8.10" - python27 = "python2.7" - python36 = "python3.6" - java8 = "java8" - go1x = "go1.x" - dotnetcore20 = "dotnetcore2.0" - dotnetcore21 = "dotnetcore2.1" - - @classmethod - def has_value(cls, value): - """ - Checks if the enum has this value - - :param string value: Value to check - :return bool: True, if enum has the value - """ - return any(value == item.value for item in cls) +LOG = logging.getLogger(__name__) class LambdaContainer(Container): @@ -46,29 +27,43 @@ class LambdaContainer(Container): # This is the dictionary that represents where the debugger_path arg is mounted in docker to as readonly. _DEBUGGER_VOLUME_MOUNT = {"bind": _DEBUGGER_VOLUME_MOUNT_PATH, "mode": "ro"} - def __init__(self, + def __init__(self, # pylint: disable=R0914 runtime, handler, code_dir, + layers, + image_builder, memory_mb=128, env_vars=None, debug_options=None): """ Initializes the class - :param string runtime: Name of the Lambda runtime - :param string handler: Handler of the function to run - :param string code_dir: Directory where the Lambda function code is present. This directory will be mounted + Parameters + ---------- + runtime str + Name of the Lambda runtime + handler str + Handler of the function to run + code_dir str + Directory where the Lambda function code is present. This directory will be mounted to the container to execute - :param int memory_mb: Optional. Max limit of memory in MegaBytes this Lambda function can use. - :param dict env_vars: Optional. Dictionary containing environment variables passed to container - :param DebugContext debug_options: Optional. Contains container debugging info (port, debugger path) + layers list(str) + List of layers + image_builder samcli.local.docker.lambda_image.LambdaImage + LambdaImage that can be used to build the image needed for starting the container + memory_mb int + Optional. Max limit of memory in MegaBytes this Lambda function can use. + env_vars dict + Optional. Dictionary containing environment variables passed to container + debug_options DebugContext + Optional. Contains container debugging info (port, debugger path) """ if not Runtime.has_value(runtime): raise ValueError("Unsupported Lambda runtime {}".format(runtime)) - image = LambdaContainer._get_image(runtime) + image = LambdaContainer._get_image(image_builder, runtime, layers) ports = LambdaContainer._get_exposed_ports(debug_options) entry = LambdaContainer._get_entry_point(runtime, debug_options) additional_options = LambdaContainer._get_additional_options(runtime, debug_options) @@ -141,14 +136,25 @@ def _get_additional_volumes(debug_options): } @staticmethod - def _get_image(runtime): + def _get_image(image_builder, runtime, layers): """ Returns the name of Docker Image for the given runtime - :param string runtime: Name of the runtime - :return: Name of Docker Image for the given runtime + Parameters + ---------- + image_builder samcli.local.docker.lambda_image.LambdaImage + LambdaImage that can be used to build the image needed for starting the container + runtime str + Name of the Lambda runtime + layers list(str) + List of layers + + Returns + ------- + str + Name of Docker Image for the given runtime """ - return "{}:{}".format(LambdaContainer._IMAGE_REPO_NAME, runtime) + return image_builder.build(runtime, layers) @staticmethod def _get_entry_point(runtime, debug_options=None): @@ -167,8 +173,13 @@ def _get_entry_point(runtime, debug_options=None): if not debug_options: return None + if runtime not in LambdaContainer._supported_runtimes(): + raise DebuggingNotSupported( + "Debugging is not currently supported for {}".format(runtime)) + debug_port = debug_options.debug_port debug_args_list = [] + if debug_options.debug_args: debug_args_list = debug_options.debug_args.split(" ") @@ -275,3 +286,12 @@ def _get_entry_point(runtime, debug_options=None): ] return entrypoint + + @staticmethod + def _supported_runtimes(): + return {Runtime.java8.value, Runtime.go1x.value, Runtime.nodejs.value, Runtime.nodejs43.value, + Runtime.nodejs610.value, Runtime.nodejs810.value, Runtime.python27.value, Runtime.python36.value} + + +class DebuggingNotSupported(Exception): + pass diff --git a/samcli/local/docker/lambda_image.py b/samcli/local/docker/lambda_image.py new file mode 100644 index 0000000000..fdcb02f0ac --- /dev/null +++ b/samcli/local/docker/lambda_image.py @@ -0,0 +1,226 @@ +""" +Generates a Docker Image to be used for invoking a function locally +""" +from enum import Enum +import uuid +import logging +import hashlib + +import docker + +from samcli.commands.local.cli_common.user_exceptions import ImageBuildException +from samcli.lib.utils.tar import create_tarball + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +LOG = logging.getLogger(__name__) + + +class Runtime(Enum): + nodejs = "nodejs" + nodejs43 = "nodejs4.3" + nodejs610 = "nodejs6.10" + nodejs810 = "nodejs8.10" + python27 = "python2.7" + python36 = "python3.6" + python37 = "python3.7" + ruby25 = "ruby2.5" + java8 = "java8" + go1x = "go1.x" + dotnetcore20 = "dotnetcore2.0" + dotnetcore21 = "dotnetcore2.1" + provided = "provided" + + @classmethod + def has_value(cls, value): + """ + Checks if the enum has this value + + :param string value: Value to check + :return bool: True, if enum has the value + """ + return any(value == item.value for item in cls) + + +class LambdaImage(object): + _LAYERS_DIR = "/opt" + _DOCKER_LAMBDA_REPO_NAME = "lambci/lambda" + _SAM_CLI_REPO_NAME = "samcli/lambda" + + def __init__(self, layer_downloader, skip_pull_image, force_image_build, docker_client=None): + """ + + Parameters + ---------- + layer_downloader samcli.local.layers.layer_downloader.LayerDownloader + LayerDownloader to download layers locally + skip_pull_image bool + True if the image should not be pulled from DockerHub + force_image_build bool + True to download the layer and rebuild the image even if it exists already on the system + docker_client docker.DockerClient + Optional docker client object + """ + self.layer_downloader = layer_downloader + self.skip_pull_image = skip_pull_image + self.force_image_build = force_image_build + self.docker_client = docker_client or docker.from_env() + + def build(self, runtime, layers): + """ + Build the image if one is not already on the system that matches the runtime and layers + + Parameters + ---------- + runtime str + Name of the Lambda runtime + layers list(samcli.commands.local.lib.provider.Layer) + List of layers + + Returns + ------- + str + The image to be used (REPOSITORY:TAG) + """ + base_image = "{}:{}".format(self._DOCKER_LAMBDA_REPO_NAME, runtime) + + # Don't build the image if there are no layers. + if not layers: + LOG.debug("Skipping building an image since no layers were defined") + return base_image + + downloaded_layers = self.layer_downloader.download_all(layers, self.force_image_build) + + docker_image_version = self._generate_docker_image_version(downloaded_layers, runtime) + image_tag = "{}:{}".format(self._SAM_CLI_REPO_NAME, docker_image_version) + + image_not_found = False + + try: + self.docker_client.images.get(image_tag) + except docker.errors.ImageNotFound: + LOG.info("Image was not found.") + image_not_found = True + + if self.force_image_build or \ + image_not_found or \ + any(layer.is_defined_within_template for layer in downloaded_layers): + LOG.info("Building image...") + self._build_image(base_image, image_tag, downloaded_layers) + + return image_tag + + @staticmethod + def _generate_docker_image_version(layers, runtime): + """ + Generate the Docker TAG that will be used to create the image + + Parameters + ---------- + layers list(samcli.commands.local.lib.provider.Layer) + List of the layers + + runtime str + Runtime of the image to create + + Returns + ------- + str + String representing the TAG to be attached to the image + """ + + # Docker has a concept of a TAG on an image. This is plus the REPOSITORY is a way to determine + # a version of the image. We will produced a TAG for a combination of the runtime with the layers + # specified in the template. This will allow reuse of the runtime and layers across different + # functions that are defined. If two functions use the same runtime with the same layers (in the + # same order), SAM CLI will only produce one image and use this image across both functions for invoke. + return runtime + '-' + hashlib.sha256( + "-".join([layer.name for layer in layers]).encode('utf-8')).hexdigest()[0:25] + + def _build_image(self, base_image, docker_tag, layers): + """ + Builds the image + + Parameters + ---------- + base_image str + Base Image to use for the new image + docker_tag + Docker tag (REPOSITORY:TAG) to use when building the image + layers list(samcli.commands.local.lib.provider.Layer) + List of Layers to be use to mount in the image + + Returns + ------- + None + + Raises + ------ + samcli.commands.local.cli_common.user_exceptions.ImageBuildException + When docker fails to build the image + """ + dockerfile_content = self._generate_dockerfile(base_image, layers) + + # Create dockerfile in the same directory of the layer cache + dockerfile_name = "dockerfile_" + str(uuid.uuid4()) + full_dockerfile_path = Path(self.layer_downloader.layer_cache, dockerfile_name) + + try: + with open(str(full_dockerfile_path), "w") as dockerfile: + dockerfile.write(dockerfile_content) + + tar_paths = {str(full_dockerfile_path): "Dockerfile"} + for layer in layers: + tar_paths[layer.codeuri] = '/' + layer.name + + with create_tarball(tar_paths) as tarballfile: + try: + self.docker_client.images.build(fileobj=tarballfile, + custom_context=True, + rm=True, + encoding='gzip', + tag=docker_tag, + pull=not self.skip_pull_image) + except (docker.errors.BuildError, docker.errors.APIError): + LOG.exception("Failed to build Docker Image") + raise ImageBuildException("Building Image failed.") + finally: + if full_dockerfile_path.exists(): + full_dockerfile_path.unlink() + + @staticmethod + def _generate_dockerfile(base_image, layers): + """ + Generate the Dockerfile contents + + A generated Dockerfile will look like the following: + ``` + FROM lambci/lambda:python3.6 + + ADD --chown=sbx_user1051:495 layer1 /opt + ADD --chown=sbx_user1051:495 layer2 /opt + ``` + + Parameters + ---------- + base_image str + Base Image to use for the new image + layers list(samcli.commands.local.lib.provider.Layer) + List of Layers to be use to mount in the image + + Returns + ------- + str + String representing the Dockerfile contents for the image + + """ + dockerfile_content = "FROM {}\n".format(base_image) + + for layer in layers: + dockerfile_content = dockerfile_content + \ + "ADD --chown=sbx_user1051:495 {} {}\n".format(layer.name, LambdaImage._LAYERS_DIR) + return dockerfile_content diff --git a/samcli/local/docker/manager.py b/samcli/local/docker/manager.py index db850195fd..951b04fbfa 100644 --- a/samcli/local/docker/manager.py +++ b/samcli/local/docker/manager.py @@ -51,7 +51,11 @@ def run(self, container, input_data=None, warm=False): is_image_local = self.has_image(image_name) - if not is_image_local or not self.skip_pull_image: + # Skip Pulling a new image if: a) Image name is samcli/lambda OR b) Image is available AND + # c) We are asked to skip pulling the image + if (is_image_local and self.skip_pull_image) or image_name.startswith('samcli/lambda'): + LOG.info("Requested to skip pulling images ...\n") + else: try: self.pull_image(image_name) except DockerImagePullFailedException: @@ -61,8 +65,6 @@ def run(self, container, input_data=None, warm=False): LOG.info( "Failed to download a new %s image. Invoking with the already downloaded image.", image_name) - else: - LOG.info("Requested to skip pulling images ...\n") if not container.is_created(): # Create the container first before running. diff --git a/samcli/local/init/__init__.py b/samcli/local/init/__init__.py index c068c20e90..8ef8c8dd26 100644 --- a/samcli/local/init/__init__.py +++ b/samcli/local/init/__init__.py @@ -14,9 +14,11 @@ _templates = os.path.join(_init_path, 'templates') RUNTIME_TEMPLATE_MAPPING = { + "python3.7": os.path.join(_templates, "cookiecutter-aws-sam-hello-python"), "python3.6": os.path.join(_templates, "cookiecutter-aws-sam-hello-python"), "python2.7": os.path.join(_templates, "cookiecutter-aws-sam-hello-python"), "python": os.path.join(_templates, "cookiecutter-aws-sam-hello-python"), + "ruby2.5": os.path.join(_templates, "cookiecutter-aws-sam-hello-ruby"), "nodejs6.10": os.path.join(_templates, "cookiecutter-aws-sam-hello-nodejs"), "nodejs8.10": os.path.join(_templates, "cookiecutter-aws-sam-hello-nodejs"), "nodejs4.3": os.path.join(_templates, "cookiecutter-aws-sam-hello-nodejs"), diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/cookiecutter.json b/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/cookiecutter.json index 1b996b510e..3599f3691c 100644 --- a/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/cookiecutter.json +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/cookiecutter.json @@ -1,4 +1,4 @@ { "project_name": "Name of the project", - "runtime": "python3.6" + "runtime": "python3.7" } \ No newline at end of file diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/README.md b/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/README.md index 47ad785c36..bf0d7ee02b 100644 --- a/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/README.md +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/README.md @@ -18,11 +18,11 @@ This is a sample template for {{ cookiecutter.project_name }} - Below is a brief ## Requirements -* AWS CLI already configured with Administrator permission -{%- if cookiecutter.runtime == 'python3.6' %} -* [Python 3 installed](https://www.python.org/downloads/) -{%- else %} +* AWS CLI already configured with at least PowerUser permission +{%- if cookiecutter.runtime == 'python2.7' %} * [Python 2.7 installed](https://www.python.org/downloads/) +{%- else %} +* [Python 3 installed](https://www.python.org/downloads/) {%- endif %} * [Docker installed](https://www.docker.com/community-edition) * [Python Virtual Environment](http://docs.python-guide.org/en/latest/dev/virtualenvs/) @@ -129,26 +129,26 @@ python -m pytest tests/ -v ### Python Virtual environment -{%- if cookiecutter.runtime == 'python3.6' %} -**In case you're new to this**, python3 comes with `virtualenv` library by default so you can simply run the following: +{%- if cookiecutter.runtime == 'python2.7' %} +**In case you're new to this**, python2 `virtualenv` module is not available in the standard library so we need to install it and then we can install our dependencies: 1. Create a new virtual environment 2. Install dependencies in the new virtual environment ```bash -python3 -m venv .venv +pip install virtualenv +virtualenv .venv . .venv/bin/activate pip install -r requirements.txt ``` {%- else %} -**In case you're new to this**, python2 `virtualenv` module is not available in the standard library so we need to install it and then we can install our dependencies: +**In case you're new to this**, python3 comes with `virtualenv` library by default so you can simply run the following: 1. Create a new virtual environment 2. Install dependencies in the new virtual environment ```bash -pip install virtualenv -virtualenv .venv +python3 -m venv .venv . .venv/bin/activate pip install -r requirements.txt ``` diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/template.yaml b/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/template.yaml index 0037ccc7ee..d709a8695a 100644 --- a/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/template.yaml +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-python/{{cookiecutter.project_name}}/template.yaml @@ -20,8 +20,10 @@ Resources: Handler: app.lambda_handler {%- if cookiecutter.runtime == 'python2.7' %} Runtime: python2.7 - {%- else %} + {%- elif cookiecutter.runtime == 'python3.6' %} Runtime: python3.6 + {%- else %} + Runtime: python3.7 {%- endif %} Environment: # More info about Env Vars: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#environment-object Variables: diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/.gitignore b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/.gitignore new file mode 100644 index 0000000000..74ea25e0e5 --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/.gitignore @@ -0,0 +1,168 @@ + +# Created by https://www.gitignore.io/api/osx,linux,python,windows + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + + +# End of https://www.gitignore.io/api/osx,linux,python,windows \ No newline at end of file diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/LICENSE b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/LICENSE new file mode 100644 index 0000000000..f19aaa6d09 --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/LICENSE @@ -0,0 +1,14 @@ +MIT No Attribution + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/README.md b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/README.md new file mode 100644 index 0000000000..f13c3e7088 --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/README.md @@ -0,0 +1,62 @@ +# Cookiecutter SAM for Ruby Lambda functions + +This is a [Cookiecutter](https://github.com/audreyr/cookiecutter) template to create a Serverless Hello World App based on Serverless Application Model (SAM) and Python. + +## Recommendations + +This cookiecutter template is easily accessible via `sam` + +``` +sam init --runtime=ruby2.5 +``` + +This should create a directory structure called ``sam-app`` with the following output. + +```bash +[+] Initializing project structure... +[SUCCESS] - Read sam-app/README.md for further instructions on how to proceed +[*] Project initialization is now complete +``` + +The directory `sam-app` contains a `README.md` with detailed instructions. + +## Alternative Installation + +You can also use `cookiecutter` CLI instead as ``{{cookiecutter.project_name}}`` will be rendered based on your input and therefore all variables and files will be rendered properly. + +### Requirements + +Install `cookiecutter` command line: + +**Pip users**: + +* `pip install cookiecutter` + +**Homebrew users**: + +* `brew install cookiecutter` + +**Windows or Pipenv users**: + +* `pipenv install cookiecutter` + +**NOTE**: [`Pipenv`](https://github.com/pypa/pipenv) is the new and recommended Python packaging tool that works across multiple platforms and makes Windows a first-class citizen. + +### Usage + +Generate a new SAM based Serverless App: `cookiecutter gh:aws-samples/cookiecutter-aws-sam-hello-ruby`. + +You'll be prompted a few questions to help this cookiecutter template to scaffold this project and after its completed you should see a new folder at your current path with the name of the project you gave as input. + +**NOTE**: After you understand how cookiecutter works (cookiecutter.json, mainly), you can fork this repo and apply your own mechanisms to accelerate your development process and this can be followed for any programming language and OS. + + +## Credits + +* This project has been generated with [Cookiecutter](https://github.com/audreyr/cookiecutter) + + +License +------- + +This project is licensed under the terms of the [MIT License with no attribution](/LICENSE) diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/cookiecutter.json b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/cookiecutter.json new file mode 100644 index 0000000000..3039309676 --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/cookiecutter.json @@ -0,0 +1,4 @@ +{ + "project_name": "Name of the project", + "runtime": "ruby2.5" +} diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/tests/test_cookiecutter.py b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/tests/test_cookiecutter.py new file mode 100644 index 0000000000..da36d53a9a --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/tests/test_cookiecutter.py @@ -0,0 +1,39 @@ +""" + Tests cookiecutter baking process and rendered content +""" + + +def test_project_tree(cookies): + result = cookies.bake(extra_context={ + 'project_name': 'hello sam' + }) + assert result.exit_code == 0 + assert result.exception is None + assert result.project.basename == 'hello sam' + assert result.project.isdir() + assert result.project.join('.gitignore').isfile() + assert result.project.join('template.yaml').isfile() + assert result.project.join('README.md').isfile() + assert result.project.join('hello_world').isdir() + assert result.project.join('hello_world', 'app.rb').isfile() + assert result.project.join('tests').isdir() + assert result.project.join('tests', 'unit', 'test_handler.py').isfile() + + +def test_app_content(cookies): + result = cookies.bake(extra_context={'project_name': 'my_lambda'}) + app_file = result.project.join('hello_world', 'app.rb') + app_content = app_file.readlines() + app_content = ''.join(app_content) + + contents = ( + "require 'httparty'", + "Sample pure Lambda function", + "location", + "message", + "Hello World!", + "statusCode" + ) + + for content in contents: + assert content in app_content diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/.gitignore b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/.gitignore new file mode 100644 index 0000000000..4808264dbf --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/.gitignore @@ -0,0 +1,244 @@ + +# Created by https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### OSX ### +*.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff: +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/dictionaries + +# Sensitive or high-churn files: +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.xml +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml + +# Gradle: +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ + +# Mongo Explorer plugin: +.idea/**/mongoSettings.xml + +## File-based project format: +*.iws + +## Plugin-specific files: + +# IntelliJ +/out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Ruby plugin and RubyMine +/.rakeTasks + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache/ +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule.* + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +.history + +### Windows ### +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Build folder + +*/build/* + +# End of https://www.gitignore.io/api/osx,linux,python,windows,pycharm,visualstudiocode \ No newline at end of file diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/Gemfile b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/Gemfile new file mode 100644 index 0000000000..3abba1b8b3 --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" + +gem "httparty" diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/README.md b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/README.md new file mode 100644 index 0000000000..919fa08da2 --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/README.md @@ -0,0 +1,169 @@ +# {{ cookiecutter.project_name }} + +This is a sample template for {{ cookiecutter.project_name }} - Below is a brief explanation of what we have generated for you: + +```bash +. +├── README.md <-- This instructions file +├── hello_world <-- Source code for a lambda function +│ ├── app.rb <-- Lambda function code +├── Gemfile <-- Ruby dependencies +├── template.yaml <-- SAM template +└── tests <-- Unit tests + └── unit + └── test_handler.rb +``` + +## Requirements + +* AWS CLI already configured with at least PowerUser permission +* [Ruby](https://www.ruby-lang.org/en/documentation/installation/) 2.5 installed +* [Docker installed](https://www.docker.com/community-edition) +* [Ruby Version Manager](http://rvm.io/) + +## Setup process + +### Match ruby version with docker image +For high fidelity development environment, make sure the local ruby version matches that of the docker image. To do so lets use [Ruby Version Manager](http://rvm.io/) + +Setup Ruby Version Manager from [Ruby Version Manager](http://rvm.io/) + +Run following commands + +```bash +rvm install ruby-2.5.0 +rvm use ruby-2.5.0 +rvm --default use 2.5.0 +``` + + +### Installing dependencies + +```sam-app``` comes with a Gemfile that defines the requirements and manages installing them. + +```bash +gem install bundler +bundle install +bundle install --deployment --path hello_world/vendor/bundle +``` + +* Step 1 installs ```bundler```which provides a consistent environment for Ruby projects by tracking and installing the exact gems and versions that are needed. +* Step 2 creates a Gemfile.lock that locks down the versions and creates the full dependency closure. +* Step 3 installs the gems to ```hello_world/vendor/bundle```. + +**NOTE:** As you change your dependencies during development you'll need to make sure these steps are repeated in order to execute your Lambda and/or API Gateway locally. + +### Local development + +**Invoking function locally through local API Gateway** + +```bash +sam local start-api +``` + +If the previous command ran successfully you should now be able to hit the following local endpoint to invoke your function `http://localhost:3000/hello` + +**SAM CLI** is used to emulate both Lambda and API Gateway locally and uses our `template.yaml` to understand how to bootstrap this environment (runtime, where the source code is, etc.) - The following excerpt is what the CLI will read in order to initialize an API and its routes: + +```yaml +... +Events: + HelloWorld: + Type: Api # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api + Properties: + Path: /hello + Method: get +``` + +## Packaging and deployment + +AWS Lambda Ruby runtime requires a flat folder with all dependencies including the application. SAM will use `CodeUri` property to know where to look up for both application and dependencies: + +```yaml +... + HelloWorldFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: hello_world/ + ... +``` + +Firstly, we need a `S3 bucket` where we can upload our Lambda functions packaged as ZIP before we deploy anything - If you don't have a S3 bucket to store code artifacts then this is a good time to create one: + +```bash +aws s3 mb s3://BUCKET_NAME +``` + +Next, run the following command to package our Lambda function to S3: + +```bash +sam package \ + --template-file template.yaml \ + --output-template-file packaged.yaml \ + --s3-bucket REPLACE_THIS_WITH_YOUR_S3_BUCKET_NAME +``` + +Next, the following command will create a Cloudformation Stack and deploy your SAM resources. + +```bash +sam deploy \ + --template-file packaged.yaml \ + --stack-name sam-app \ + --capabilities CAPABILITY_IAM +``` + +> **See [Serverless Application Model (SAM) HOWTO Guide](https://github.com/awslabs/serverless-application-model/blob/master/HOWTO.md) for more details in how to get started.** + +After deployment is complete you can run the following command to retrieve the API Gateway Endpoint URL: + +```bash +aws cloudformation describe-stacks \ + --stack-name sam-app \ + --query 'Stacks[].Outputs' +``` + +## Testing + +We use [Mocha](http://gofreerange.com/mocha/docs) for testing our code and you can install it using gem: ``gem install mocha`` + +Next, we run our initial unit tests: + +```bash +ruby tests/unit/test_hello.rb +``` + +**NOTE**: It is recommended to use a Ruby Version Manager to manage, and work with multiple ruby environments from interpreters to sets of gems +# Appendix + +## AWS CLI commands + +AWS CLI commands to package, deploy and describe outputs defined within the cloudformation stack: + +```bash +sam package \ + --template-file template.yaml \ + --output-template-file packaged.yaml \ + --s3-bucket REPLACE_THIS_WITH_YOUR_S3_BUCKET_NAME + +sam deploy \ + --template-file packaged.yaml \ + --stack-name sam-app \ + --capabilities CAPABILITY_IAM \ + --parameter-overrides MyParameterSample=MySampleValue + +aws cloudformation describe-stacks \ + --stack-name sam-app --query 'Stacks[].Outputs' +``` + +## Bringing to the next level + +Here are a few ideas that you can use to get more acquainted as to how this overall process works: + +* Create an additional API resource (e.g. /hello/{proxy+}) and return the name requested through this new path +* Update unit test to capture that +* Package & Deploy + +Next, you can use the following resources to know more about beyond hello world samples and how others structure their Serverless applications: + +* [AWS Serverless Application Repository](https://aws.amazon.com/serverless/serverlessrepo/) + diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/hello_world/app.rb b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/hello_world/app.rb new file mode 100644 index 0000000000..89f645d0ff --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/hello_world/app.rb @@ -0,0 +1,83 @@ +require 'httparty' +require 'json' + +def lambda_handler(event:, context:) + # Sample pure Lambda function + + # Parameters + # ---------- + # event: Hash, required + # API Gateway Lambda Proxy Input Format + + # { + # "resource": "Resource path", + # "path": "Path parameter", + # "httpMethod": "Incoming request's method name" + # "headers": {Incoming request headers} + # "queryStringParameters": {query string parameters } + # "pathParameters": {path parameters} + # "stageVariables": {Applicable stage variables} + # "requestContext": {Request context, including authorizer-returned key-value pairs} + # "body": "A JSON string of the request payload." + # "isBase64Encoded": "A boolean flag to indicate if the applicable request payload is Base64-encode" + # } + + # https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format + + # context: object, required + # Lambda Context runtime methods and attributes + + # Attributes + # ---------- + + # context.aws_request_id: str + # Lambda request ID + # context.client_context: object + # Additional context when invoked through AWS Mobile SDK + # context.function_name: str + # Lambda function name + # context.function_version: str + # Function version identifier + # context.get_remaining_time_in_millis: function + # Time in milliseconds before function times out + # context.identity: + # Cognito identity provider context when invoked through AWS Mobile SDK + # context.invoked_function_arn: str + # Function ARN + # context.log_group_name: str + # Cloudwatch Log group name + # context.log_stream_name: str + # Cloudwatch Log stream name + # context.memory_limit_in_mb: int + # Function memory + + # Returns + # ------ + # API Gateway Lambda Proxy Output Format: dict + # 'statusCode' and 'body' are required + + # { + # "isBase64Encoded": true | false, + # "statusCode": httpStatusCode, + # "headers": {"headerName": "headerValue", ...}, + # "body": "..." + # } + + # # api-gateway-simple-proxy-for-lambda-output-format + # https: // docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html + + begin + response = HTTParty.get('http://checkip.amazonaws.com/') + rescue HTTParty::Error => error + puts error.inspect + raise error + end + + return { + :statusCode => response.code, + :body => { + :message => "Hello World!", + :location => response.body + }.to_json + } +end diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/template.yaml b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/template.yaml new file mode 100644 index 0000000000..5617c03472 --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/template.yaml @@ -0,0 +1,44 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + {{ cookiecutter.project_name }} + + Sample SAM Template for {{ cookiecutter.project_name }} + +# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst +Globals: + Function: + Timeout: 3 + + +Resources: + + HelloWorldFunction: + Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction + Properties: + CodeUri: hello_world/ + Handler: app.lambda_handler + Runtime: ruby2.5 + Environment: # More info about Env Vars: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#environment-object + Variables: + PARAM1: VALUE + Events: + HelloWorld: + Type: Api # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api + Properties: + Path: /hello + Method: get + +Outputs: + + HelloWorldApi: + Description: "API Gateway endpoint URL for Prod stage for Hello World function" + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/hello/" + + HelloWorldFunction: + Description: "Hello World Lambda Function ARN" + Value: !GetAtt HelloWorldFunction.Arn + + HelloWorldFunctionIamRole: + Description: "Implicit IAM Role created for Hello World function" + Value: !GetAtt HelloWorldFunctionRole.Arn diff --git a/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/tests/unit/test_handler.rb b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/tests/unit/test_handler.rb new file mode 100644 index 0000000000..8ae3a2595e --- /dev/null +++ b/samcli/local/init/templates/cookiecutter-aws-sam-hello-ruby/{{cookiecutter.project_name}}/tests/unit/test_handler.rb @@ -0,0 +1,91 @@ +require 'json' +require 'test/unit' +require 'mocha/test_unit' +require_relative '../../hello_world/app' + +class HelloWorldTest < Test::Unit::TestCase + + def setup + @event = { + body: 'eyJ0ZXN0IjoiYm9keSJ9', + resource: '/{proxy+}', + path: '/path/to/resource', + httpMethod: 'POST', + isBase64Encoded: true, + queryStringParameters: { + foo: 'bar' + }, + pathParameters: { + proxy: '/path/to/resource' + }, + stageVariables: { + baz: 'qux' + }, + headers: { + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + Accept-Encoding: 'gzip, deflate, sdch', + Accept-Language: 'en-US,en;q=0.8', + Cache-Control: 'max-age=0', + CloudFront-Forwarded-Proto: 'https', + CloudFront-Is-Desktop-Viewer: 'true', + CloudFront-Is-Mobile-Viewer: 'false', + CloudFront-Is-SmartTV-Viewer: 'false', + CloudFront-Is-Tablet-Viewer: 'false', + CloudFront-Viewer-Country: 'US', + Host: '1234567890.execute-api.us-east-1.amazonaws.com', + Upgrade-Insecure-Requests: '1', + User-Agent: 'Custom User Agent String', + Via: '1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)', + X-Amz-Cf-Id: 'cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==', + X-Forwarded-For: '127.0.0.1, 127.0.0.2', + X-Forwarded-Port: '443', + X-Forwarded-Proto: 'https' + }, + requestContext: { + accountId: '123456789012', + resourceId: '123456', + stage: 'prod', + requestId: 'c6af9ac6-7b61-11e6-9a41-93e8deadbeef', + requestTime: '09/Apr/2015:12:34:56 +0000', + requestTimeEpoch: 1428582896000, + identity: { + cognitoIdentityPoolId: "null", + accountId: "null", + cognitoIdentityId: "null", + caller: "null", + accessKey: "null", + sourceIp: '127.0.0.1', + cognitoAuthenticationType: "null", + cognitoAuthenticationProvider: "null", + userArn: "null", + userAgent: 'Custom User Agent String', + user: "null" + }, + path: '/prod/path/to/resource', + resourcePath: '/{proxy+}', + httpMethod: 'POST', + apiId: '1234567890', + protocol: 'HTTP/1.1' + } + } + + @mock_response = { + :statusCode => 200, + :body => { + message: "Hello World!", + location: "1.1.1.1" + }.to_json + } + + end + + def test_lambda_handler + expects(:lambda_handler).with(event:@event, context:"").returns(@mock_response) + response = lambda_handler(event:@event, context:"") + json_body = JSON.parse(response[:body]) + + assert_equal(200, response[:statusCode]) + assert_equal("Hello World!", json_body["message"]) + assert_equal("1.1.1.1", json_body["location"]) + end +end diff --git a/samcli/local/lambdafn/config.py b/samcli/local/lambdafn/config.py index a1194a0141..40174ce935 100644 --- a/samcli/local/lambdafn/config.py +++ b/samcli/local/lambdafn/config.py @@ -19,26 +19,38 @@ def __init__(self, runtime, handler, code_abs_path, + layers, memory=None, timeout=None, env_vars=None): """ Initialize the class. - :param string name: Name of the function - :param string runtime: Runtime of function - :param string handler: Handler method - :param string code_abs_path: Absolute path to the code - :param integer memory: Function memory limit in MB - :param integer timeout: Function timeout in seconds - :param samcli.local.lambdafn.env_vars.EnvironmentVariables env_vars: Optional, Environment variables. + Parameters + ---------- + name str + Name of the function + runtime str + Runtime of function + handler str + Handler method + code_abs_path str + Absolute path to the code + layers list(str) + List of Layers + memory int + Function memory limit in MB + timeout int + Function timeout in seconds + env_vars samcli.local.lambdafn.env_vars.EnvironmentVariables + Optional, Environment variables. If it not provided, this class will generate one for you based on the function properties """ - self.name = name self.runtime = runtime self.handler = handler self.code_abs_path = code_abs_path + self.layers = layers self.memory = memory or self._DEFAULT_MEMORY self.timeout = timeout or self._DEFAULT_TIMEOUT_SECONDS diff --git a/samcli/local/lambdafn/runtime.py b/samcli/local/lambdafn/runtime.py index 4f55134aab..ed0e564933 100644 --- a/samcli/local/lambdafn/runtime.py +++ b/samcli/local/lambdafn/runtime.py @@ -25,14 +25,19 @@ class LambdaRuntime(object): SUPPORTED_ARCHIVE_EXTENSIONS = (".zip", ".jar", ".ZIP", ".JAR") - def __init__(self, container_manager): + def __init__(self, container_manager, image_builder): """ Initialize the Local Lambda runtime - :param samcli.local.docker.manager.ContainerManager container_manager: Instance of the ContainerManager class - that can run a local Docker container + Parameters + ---------- + container_manager samcli.local.docker.manager.ContainerManager + Instance of the ContainerManager class that can run a local Docker container + image_builder samcli.local.docker.lambda_image.LambdaImage + Instance of the LambdaImage class that can create am image """ self._container_manager = container_manager + self._image_builder = image_builder def invoke(self, function_config, @@ -69,6 +74,8 @@ def invoke(self, container = LambdaContainer(function_config.runtime, function_config.handler, code_dir, + function_config.layers, + self._image_builder, memory_mb=function_config.memory, env_vars=env_vars, debug_options=debug_context) diff --git a/samcli/local/lambdafn/zip.py b/samcli/local/lambdafn/zip.py index ac178afaf3..856dcbc4a4 100644 --- a/samcli/local/lambdafn/zip.py +++ b/samcli/local/lambdafn/zip.py @@ -7,10 +7,20 @@ import zipfile import logging +import requests + +from samcli.lib.utils.progressbar import progressbar + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + LOG = logging.getLogger(__name__) -def unzip(zip_file_path, output_dir): +def unzip(zip_file_path, output_dir, permission=None): """ Unzip the given file into the given directory while preserving file permissions in the process. @@ -21,6 +31,9 @@ def unzip(zip_file_path, output_dir): output_dir : str Path to the directory where the it should be unzipped to + + permission : octal int + Permission to set """ with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: @@ -33,6 +46,26 @@ def unzip(zip_file_path, output_dir): zip_ref.extract(name, output_dir) _set_permissions(file_info, extracted_path) + _override_permissions(extracted_path, permission) + + _override_permissions(output_dir, permission) + + +def _override_permissions(path, permission): + """ + Forcefully override the permissions on the path + + Parameters + ---------- + path str + Path where the file or directory + permission octal int + Permission to set + + """ + if permission: + os.chmod(path, permission) + def _set_permissions(zip_file_info, extracted_path): """ @@ -56,3 +89,42 @@ def _set_permissions(zip_file_info, extracted_path): return os.chmod(extracted_path, permission) + + +def unzip_from_uri(uri, layer_zip_path, unzip_output_dir, progressbar_label): + """ + Download the LayerVersion Zip to the Layer Pkg Cache + + Parameters + ---------- + uri str + Uri to download from + layer_zip_path str + Path to where the content from the uri should be downloaded to + unzip_output_dir str + Path to unzip the zip to + progressbar_label str + Label to use in the Progressbar + """ + try: + get_request = requests.get(uri, stream=True) + + with open(layer_zip_path, 'wb') as local_layer_file: + file_length = int(get_request.headers['Content-length']) + + with progressbar(file_length, progressbar_label) as p_bar: + # Set the chunk size to None. Since we are streaming the request, None will allow the data to be + # read as it arrives in whatever size the chunks are received. + for data in get_request.iter_content(chunk_size=None): + local_layer_file.write(data) + p_bar.update(len(data)) + + # Forcefully set the permissions to 700 on files and directories. This is to ensure the owner + # of the files is the only one that can read, write, or execute the files. + unzip(layer_zip_path, unzip_output_dir, permission=0o700) + + finally: + # Remove the downloaded zip file + path_to_layer = Path(layer_zip_path) + if path_to_layer.exists(): + path_to_layer.unlink() diff --git a/samcli/local/layers/__init__.py b/samcli/local/layers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samcli/local/layers/layer_downloader.py b/samcli/local/layers/layer_downloader.py new file mode 100644 index 0000000000..fe2422b51c --- /dev/null +++ b/samcli/local/layers/layer_downloader.py @@ -0,0 +1,177 @@ +""" +Downloads Layers locally +""" + +import logging + +import boto3 +from botocore.exceptions import NoCredentialsError, ClientError + +from samcli.lib.utils.codeuri import resolve_code_path +from samcli.local.lambdafn.zip import unzip_from_uri +from samcli.commands.local.cli_common.user_exceptions import CredentialsRequired, ResourceNotFound + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +LOG = logging.getLogger(__name__) + + +class LayerDownloader(object): + + def __init__(self, layer_cache, cwd, lambda_client=None): + """ + + Parameters + ---------- + layer_cache str + path where to cache layers + cwd str + Current working directory + lambda_client boto3.client('lambda') + Boto3 Client for AWS Lambda + """ + self.layer_cache = layer_cache + self.cwd = cwd + self.lambda_client = lambda_client or boto3.client('lambda') + + def download_all(self, layers, force=False): + """ + Download a list of layers to the cache + + Parameters + ---------- + layers list(samcli.commands.local.lib.provider.Layer) + List of Layers representing the layer to be downloaded + force bool + True to download the layer even if it exists already on the system + + Returns + ------- + List(Path) + List of Paths to where the layer was cached + """ + layer_dirs = [] + for layer in layers: + layer_dirs.append(self.download(layer, force)) + + return layer_dirs + + def download(self, layer, force=False): + """ + Download a given layer to the local cache. + + Parameters + ---------- + layer samcli.commands.local.lib.provider.Layer + Layer representing the layer to be downloaded. + force bool + True to download the layer even if it exists already on the system + + Returns + ------- + Path + Path object that represents where the layer is download to + """ + if layer.is_defined_within_template: + LOG.info("%s is a local Layer in the template", layer.name) + layer.codeuri = resolve_code_path(self.cwd, layer.codeuri) + return layer + + LayerDownloader._create_cache(self.layer_cache) + + # disabling no-member due to https://github.com/PyCQA/pylint/issues/1660 + layer_path = Path(self.layer_cache).joinpath(layer.name).resolve() # pylint: disable=no-member + is_layer_downloaded = self._is_layer_cached(layer_path) + layer.codeuri = str(layer_path) + + if is_layer_downloaded and not force: + LOG.info("%s is already cached. Skipping download", layer.arn) + return layer + + layer_zip_path = layer.codeuri + '.zip' + layer_zip_uri = self._fetch_layer_uri(layer) + unzip_from_uri(layer_zip_uri, + layer_zip_path, + unzip_output_dir=layer.codeuri, + progressbar_label='Downloading {}'.format(layer.layer_arn)) + + return layer + + def _fetch_layer_uri(self, layer): + """ + Fetch the Layer Uri based on the LayerVersion Arn + + Parameters + ---------- + layer samcli.commands.local.lib.provider.LayerVersion + LayerVersion to fetch + + Returns + ------- + str + The Uri to download the LayerVersion Content from + + Raises + ------ + samcli.commands.local.cli_common.user_exceptions.NoCredentialsError + When the Credentials given are not sufficient to call AWS Lambda + """ + try: + layer_version_response = self.lambda_client.get_layer_version(LayerName=layer.layer_arn, + VersionNumber=layer.version) + except NoCredentialsError: + raise CredentialsRequired("Layers require credentials to download the layers locally.") + except ClientError as e: + error_code = e.response.get('Error').get('Code') + error_exc = { + 'AccessDeniedException': CredentialsRequired( + "Credentials provided are missing lambda:Getlayerversion policy that is needed to download the " + "layer or you do not have permission to download the layer"), + 'ResourceNotFoundException': ResourceNotFound("{} was not found.".format(layer.arn)) + } + + if error_code in error_exc: + raise error_exc[error_code] + + # If it was not 'AccessDeniedException' or 'ResourceNotFoundException' re-raise + raise e + + return layer_version_response.get("Content").get("Location") + + def _is_layer_cached(self, layer_path): + """ + Checks if the layer is already cached on the system + + Parameters + ---------- + layer_path Path + Path to where the layer should exist if cached on the system + + Returns + ------- + bool + True if the layer_path already exists otherwise False + + """ + return layer_path.exists() + + @staticmethod + def _create_cache(layer_cache): + """ + Create the Cache directory if it does not exist. + + Parameters + ---------- + layer_cache + Directory to where the layers should be cached + + Returns + ------- + None + + """ + Path(layer_cache).mkdir(mode=0o700, parents=True, exist_ok=True) diff --git a/tests/functional/commands/local/lib/test_local_api_service.py b/tests/functional/commands/local/lib/test_local_api_service.py index d8a77598f2..5065f5a401 100644 --- a/tests/functional/commands/local/lib/test_local_api_service.py +++ b/tests/functional/commands/local/lib/test_local_api_service.py @@ -15,6 +15,8 @@ from samcli.local.lambdafn.runtime import LambdaRuntime from samcli.local.docker.manager import ContainerManager from samcli.commands.local.lib.local_api_service import LocalApiService +from samcli.local.layers.layer_downloader import LayerDownloader +from samcli.local.docker.lambda_image import LambdaImage from tests.functional.function_code import nodejs_lambda, API_GATEWAY_ECHO_EVENT from unittest import TestCase @@ -49,7 +51,7 @@ def setUp(self): self.function = provider.Function(name=self.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=self.code_uri, environment={}, - rolearn=None) + rolearn=None, layers=[]) self.mock_function_provider = Mock() self.mock_function_provider.get.return_value = self.function @@ -64,9 +66,11 @@ def setUp(self): # Now wire up the Lambda invoker and pass it through the context self.lambda_invoke_context_mock = Mock() manager = ContainerManager() - local_runtime = LambdaRuntime(manager) + layer_downloader = LayerDownloader("./", "./") + lambda_image = LambdaImage(layer_downloader, False) + local_runtime = LambdaRuntime(manager, lambda_image) lambda_runner = LocalLambdaRunner(local_runtime, self.mock_function_provider, self.cwd, env_vars_values=None, - debug_context=None, aws_profile=None) + debug_context=None) self.lambda_invoke_context_mock.local_lambda_runner = lambda_runner self.lambda_invoke_context_mock.get_cwd.return_value = self.cwd diff --git a/tests/functional/commands/local/lib/test_local_lambda.py b/tests/functional/commands/local/lib/test_local_lambda.py index a4a85583e6..29c3676cd2 100644 --- a/tests/functional/commands/local/lib/test_local_lambda.py +++ b/tests/functional/commands/local/lib/test_local_lambda.py @@ -12,6 +12,8 @@ from samcli.commands.local.lib.local_lambda import LocalLambdaRunner from samcli.local.lambdafn.runtime import LambdaRuntime from samcli.local.docker.manager import ContainerManager +from samcli.local.layers.layer_downloader import LayerDownloader +from samcli.local.docker.lambda_image import LambdaImage from tests.functional.function_code import nodejs_lambda, GET_ENV_VAR from unittest import TestCase @@ -47,7 +49,7 @@ def setUp(self): self.function = provider.Function(name=self.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=self.code_uri, environment={"Variables": self.variables}, - rolearn=None) + rolearn=None, layers=[]) self.mock_function_provider = Mock() self.mock_function_provider.get.return_value = self.function @@ -65,9 +67,11 @@ def test_must_invoke(self): } manager = ContainerManager() - local_runtime = LambdaRuntime(manager) + layer_downloader = LayerDownloader("./", "./") + lambda_image = LambdaImage(layer_downloader, False) + local_runtime = LambdaRuntime(manager, lambda_image) runner = LocalLambdaRunner(local_runtime, self.mock_function_provider, self.cwd, self.env_var_overrides, - debug_context=None, aws_profile=None) + debug_context=None) # Append the real AWS credentials to the expected values. creds = runner.get_aws_creds() diff --git a/tests/functional/commands/validate/lib/test_sam_template_validator.py b/tests/functional/commands/validate/lib/test_sam_template_validator.py index 3f6f19a9a9..0c21cb4821 100644 --- a/tests/functional/commands/validate/lib/test_sam_template_validator.py +++ b/tests/functional/commands/validate/lib/test_sam_template_validator.py @@ -81,6 +81,28 @@ def test_valid_template_with_local_code_for_function(self): # Should not throw an exception validator.is_valid() + def test_valid_template_with_local_code_for_layer_version(self): + template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Transform": "AWS::Serverless-2016-10-31", + "Resources": { + "ServerlessLayerVersion": { + "Type": "AWS::Serverless::LayerVersion", + "Properties": { + "ContentUri": "./" + } + } + } + } + + managed_policy_mock = Mock() + managed_policy_mock.load.return_value = {"PolicyName": "FakePolicy"} + + validator = SamTemplateValidator(template, managed_policy_mock) + + # Should not throw an exception + validator.is_valid() + def test_valid_template_with_local_code_for_api(self): template = { "AWSTemplateFormatVersion": "2010-09-09", diff --git a/tests/functional/local/apigw/test_local_apigw_service.py b/tests/functional/local/apigw/test_local_apigw_service.py index 75ddd7a288..0b1fa8757c 100644 --- a/tests/functional/local/apigw/test_local_apigw_service.py +++ b/tests/functional/local/apigw/test_local_apigw_service.py @@ -15,6 +15,8 @@ from samcli.local.lambdafn.runtime import LambdaRuntime from samcli.commands.local.lib.local_lambda import LocalLambdaRunner from samcli.local.docker.manager import ContainerManager +from samcli.local.layers.layer_downloader import LayerDownloader +from samcli.local.docker.lambda_image import LambdaImage class TestService_InvalidResponses(TestCase): @@ -30,7 +32,7 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -76,11 +78,11 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.base64_response_function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -129,11 +131,11 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.base64_response_function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -347,7 +349,7 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -406,7 +408,7 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.echoimagehandler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -483,7 +485,7 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.base54request", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -563,11 +565,11 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.base64_response_function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -612,7 +614,9 @@ def test_flask_default_options_is_disabled(self): def make_service(list_of_routes, function_provider, cwd): port = random_port() manager = ContainerManager() - local_runtime = LambdaRuntime(manager) + layer_downloader = LayerDownloader("./", "./") + lambda_image = LambdaImage(layer_downloader, False) + local_runtime = LambdaRuntime(manager, lambda_image) lambda_runner = LocalLambdaRunner(local_runtime=local_runtime, function_provider=function_provider, cwd=cwd) diff --git a/tests/functional/local/docker/test_lambda_container.py b/tests/functional/local/docker/test_lambda_container.py index e80492dc93..dedc2d3954 100644 --- a/tests/functional/local/docker/test_lambda_container.py +++ b/tests/functional/local/docker/test_lambda_container.py @@ -6,7 +6,6 @@ import random import shutil import docker -import six from contextlib import contextmanager from unittest import TestCase @@ -15,6 +14,8 @@ from tests.functional.function_code import nodejs_lambda from samcli.local.docker.lambda_container import LambdaContainer from samcli.local.docker.manager import ContainerManager +from samcli.local.docker.lambda_image import LambdaImage +from samcli.local.layers.layer_downloader import LayerDownloader class TestLambdaContainer(TestCase): @@ -47,6 +48,7 @@ def setUp(self): self.runtime = "nodejs4.3" self.expected_docker_image = self.IMAGE_NAME self.handler = "index.handler" + self.layers = [] self.debug_port = _rand_port() self.debug_context = DebugContext(debug_port=self.debug_port, debugger_path=None, @@ -66,8 +68,9 @@ def test_basic_creation(self): """ A docker container must be successfully created """ - - container = LambdaContainer(self.runtime, self.handler, self.code_dir) + layer_downloader = LayerDownloader("./", "./") + image_builder = LambdaImage(layer_downloader, False) + container = LambdaContainer(self.runtime, self.handler, self.code_dir, self.layers, image_builder) self.assertIsNone(container.id, "Container must not have ID before creation") @@ -83,7 +86,9 @@ def test_basic_creation(self): def test_debug_port_is_created_on_host(self): - container = LambdaContainer(self.runtime, self.handler, self.code_dir, debug_options=self.debug_context) + layer_downloader = LayerDownloader("./", "./") + image_builder = LambdaImage(layer_downloader, False) + container = LambdaContainer(self.runtime, self.handler, self.code_dir, self.layers, image_builder, debug_options=self.debug_context) with self._create(container): @@ -96,7 +101,9 @@ def test_debug_port_is_created_on_host(self): self.assertEquals(port_binding[0]["HostPort"], str(self.debug_port)) def test_container_is_attached_to_network(self): - container = LambdaContainer(self.runtime, self.handler, self.code_dir) + layer_downloader = LayerDownloader("./", "./") + image_builder = LambdaImage(layer_downloader, False) + container = LambdaContainer(self.runtime, self.handler, self.code_dir, self.layers, image_builder) with self._network_create() as network: @@ -120,7 +127,9 @@ def test_function_result_is_available_in_stdout_and_logs_in_stderr(self): expected_output = b'{"a":"b"}' expected_stderr = b"**This string is printed from Lambda function**" - container = LambdaContainer(self.runtime, self.handler, self.code_dir) + layer_downloader = LayerDownloader("./", "./") + image_builder = LambdaImage(layer_downloader, False) + container = LambdaContainer(self.runtime, self.handler, self.code_dir, self.layers, image_builder) stdout_stream = io.BytesIO() stderr_stream = io.BytesIO() diff --git a/tests/functional/local/lambda_service/test_local_lambda_invoke.py b/tests/functional/local/lambda_service/test_local_lambda_invoke.py index 9faa07144d..84b212a106 100644 --- a/tests/functional/local/lambda_service/test_local_lambda_invoke.py +++ b/tests/functional/local/lambda_service/test_local_lambda_invoke.py @@ -15,6 +15,8 @@ from samcli.commands.local.lib.local_lambda import LocalLambdaRunner from samcli.local.docker.manager import ContainerManager from samcli.local.lambdafn.exceptions import FunctionNotFound +from samcli.local.layers.layer_downloader import LayerDownloader +from samcli.local.docker.lambda_image import LambdaImage class TestLocalLambdaService(TestCase): @@ -46,13 +48,14 @@ def setUpClass(cls): cls.hello_world_function = provider.Function(name=cls.hello_world_function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", - codeuri=cls.code_uri, environment=None, rolearn=None) + codeuri=cls.code_uri, environment=None, rolearn=None, layers=[]) cls.throw_error_function_name = "ThrowError" cls.throw_error_function = provider.Function(name=cls.throw_error_function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", - codeuri=cls.code_uri_for_throw_error, environment=None, rolearn=None) + codeuri=cls.code_uri_for_throw_error, environment=None, + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.side_effect = cls.mocked_function_provider @@ -122,7 +125,7 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -186,7 +189,7 @@ def setUpClass(cls): cls.function = provider.Function(name=cls.function_name, runtime="nodejs4.3", memory=256, timeout=5, handler="index.handler", codeuri=cls.code_uri, environment=None, - rolearn=None) + rolearn=None, layers=[]) cls.mock_function_provider = Mock() cls.mock_function_provider.get.return_value = cls.function @@ -310,7 +313,9 @@ def test_generic_405_error_when_request_path_with_invalid_method(self): def make_service(function_provider, cwd): port = random_port() manager = ContainerManager() - local_runtime = LambdaRuntime(manager) + layer_downloader = LayerDownloader("./", "./") + image_builder = LambdaImage(layer_downloader, False) + local_runtime = LambdaRuntime(manager, image_builder) lambda_runner = LocalLambdaRunner(local_runtime=local_runtime, function_provider=function_provider, cwd=cwd) diff --git a/tests/functional/local/lambdafn/test_runtime.py b/tests/functional/local/lambdafn/test_runtime.py index 47b891c4ad..6a419d37d5 100644 --- a/tests/functional/local/lambdafn/test_runtime.py +++ b/tests/functional/local/lambdafn/test_runtime.py @@ -14,6 +14,8 @@ from samcli.local.docker.manager import ContainerManager from samcli.local.lambdafn.runtime import LambdaRuntime from samcli.local.lambdafn.config import FunctionConfig +from samcli.local.layers.layer_downloader import LayerDownloader +from samcli.local.docker.lambda_image import LambdaImage logging.basicConfig(level=logging.INFO) @@ -36,7 +38,9 @@ def setUp(self): } self.container_manager = ContainerManager() - self.runtime = LambdaRuntime(self.container_manager) + layer_downloader = LayerDownloader("./", "./") + self.lambda_image = LambdaImage(layer_downloader, False) + self.runtime = LambdaRuntime(self.container_manager, self.lambda_image) def tearDown(self): for _, dir in self.code_dir.items(): @@ -51,6 +55,7 @@ def test_echo_function(self): runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir["echo"], + layers=[], timeout=timeout) stdout_stream = io.BytesIO() @@ -71,6 +76,7 @@ def test_function_timeout(self): runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir["sleep"], + layers=[], timeout=timeout) # Measure the actual duration of execution @@ -108,6 +114,7 @@ def test_echo_function_with_zip_file(self, file_name_extension): runtime=RUNTIME, handler=HANDLER, code_abs_path=code_zip_path, + layers=[], timeout=timeout) stdout_stream = io.BytesIO() @@ -144,6 +151,7 @@ def test_check_environment_variables(self): runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir["envvar"], + layers=[], memory=MEMORY, timeout=timeout) @@ -181,7 +189,9 @@ def setUp(self): random.shuffle(self.inputs) container_manager = ContainerManager() - self.runtime = LambdaRuntime(container_manager) + layer_downloader = LayerDownloader("./", "./") + self.lambda_image = LambdaImage(layer_downloader, False) + self.runtime = LambdaRuntime(container_manager, self.lambda_image) def tearDown(self): shutil.rmtree(self.code_dir) @@ -196,6 +206,7 @@ def _invoke_sleep(self, timeout, sleep_duration, check_stdout, exceptions=None): runtime=RUNTIME, handler=HANDLER, code_abs_path=self.code_dir, + layers=[], memory=1024, timeout=timeout) diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 8b15cef8ec..58e3e09d83 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -48,6 +48,14 @@ def test_with_default_requirements(self, runtime, use_container): self._verify_built_artifact(self.default_build_dir, self.FUNCTION_LOGICAL_ID, self.EXPECTED_FILES_PROJECT_MANIFEST) + self._verify_resource_property(str(self.built_template), + "OtherRelativePathResource", + "BodyS3Location", + os.path.relpath( + os.path.normpath(os.path.join(str(self.test_data_path), "SomeRelativePath")), + str(self.default_build_dir)) + ) + expected = { "pi": "3.14", "jinja": "Hello World" diff --git a/tests/integration/local/invoke/invoke_integ_base.py b/tests/integration/local/invoke/invoke_integ_base.py index 5f56a8e7dc..084f0fe846 100644 --- a/tests/integration/local/invoke/invoke_integ_base.py +++ b/tests/integration/local/invoke/invoke_integ_base.py @@ -8,17 +8,19 @@ class InvokeIntegBase(TestCase): + template = None @classmethod def setUpClass(cls): cls.cmd = cls.base_command() + cls.test_data_path = cls.get_integ_dir().joinpath("testdata") + cls.template_path = str(cls.test_data_path.joinpath("invoke", cls.template)) + cls.event_path = str(cls.test_data_path.joinpath("invoke", "event.json")) + cls.env_var_path = str(cls.test_data_path.joinpath("invoke", "vars.json")) - integration_dir = str(Path(__file__).resolve().parents[2]) - - cls.test_data_path = os.path.join(integration_dir, "testdata") - cls.template_path = os.path.join(cls.test_data_path, "invoke", "template.yml") - cls.event_path = os.path.join(cls.test_data_path, "invoke", "event.json") - cls.env_var_path = os.path.join(cls.test_data_path, "invoke", "vars.json") + @staticmethod + def get_integ_dir(): + return Path(__file__).resolve().parents[2] @classmethod def base_command(cls): @@ -29,7 +31,8 @@ def base_command(cls): return command def get_command_list(self, function_to_invoke, template_path=None, event_path=None, env_var_path=None, - parameter_overrides=None, region=None, docker_network=None): + parameter_overrides=None, region=None, no_event=None, profile=None, layer_cache=None, + docker_network=None): command_list = [self.cmd, "local", "invoke", function_to_invoke] if template_path: @@ -41,6 +44,15 @@ def get_command_list(self, function_to_invoke, template_path=None, event_path=No if env_var_path: command_list = command_list + ["-n", env_var_path] + if no_event: + command_list = command_list + ["--no-event"] + + if profile: + command_list = command_list + ["--profile", profile] + + if layer_cache: + command_list = command_list + ["--layer-cache-basedir", layer_cache] + if docker_network: command_list = command_list + ["--docker-network", docker_network] diff --git a/tests/integration/local/invoke/layer_utils.py b/tests/integration/local/invoke/layer_utils.py new file mode 100644 index 0000000000..0dd942f509 --- /dev/null +++ b/tests/integration/local/invoke/layer_utils.py @@ -0,0 +1,49 @@ +import uuid +from collections import namedtuple + +import boto3 + +from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase + +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + + +class LayerUtils(object): + + def __init__(self, region): + self.region = region + self.layer_meta = namedtuple('LayerMeta', ['layer_name', 'layer_arn', 'layer_version']) + self.lambda_client = boto3.client('lambda', + region_name=region) + self.parameters_overrides = {} + self.layers_meta = [] + self.layer_zip_parent = InvokeIntegBase.get_integ_dir().joinpath("testdata", "invoke", "layer_zips") + + @staticmethod + def generate_layer_name(): + return str(uuid.uuid4()).replace('-', '')[:10] + + def upsert_layer(self, layer_name, ref_layer_name, layer_zip): + with open(str(Path.joinpath(self.layer_zip_parent, layer_zip)), 'rb') as zip_contents: + resp = self.lambda_client.publish_layer_version( + LayerName=layer_name, + Content={ + 'ZipFile': zip_contents.read() + }) + self.parameters_overrides[ref_layer_name] = resp['LayerVersionArn'] + self.layers_meta.append( + self.layer_meta( + layer_name=layer_name, + layer_arn=resp['LayerArn'], + layer_version=resp['Version']) + ) + + def delete_layers(self): + for layer_meta in self.layers_meta: + self.lambda_client.delete_layer_version( + LayerName=layer_meta.layer_arn, + VersionNumber=layer_meta.layer_version + ) diff --git a/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py b/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py index 6cb8b23938..5e24f93204 100644 --- a/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py +++ b/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py @@ -1,3 +1,5 @@ +# coding=utf-8 + import os import tempfile @@ -6,13 +8,17 @@ from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + class TestWithDifferentLambdaRuntimeZips(InvokeIntegBase): + template = Path("runtimes", "template.yaml") def setUp(self): - self.template_path = os.path.join(self.test_data_path, "invoke", "runtimes", "template.yaml") - # Don't delete on close. Need the file to be present for tests to run. events_file = tempfile.NamedTemporaryFile(delete=False) events_file.write(b'"yolo"') # Just empty event @@ -39,3 +45,17 @@ def test_runtime_zip(self, function_name): self.assertEquals(return_code, 0) process_stdout = b"".join(process.stdout.readlines()).strip() self.assertEquals(process_stdout.decode('utf-8'), '"Hello World"') + + def test_custom_provided_runtime(self): + command_list = self.get_command_list("CustomBashFunction", + template_path=self.template_path, + event_path=self.events_file_path) + + command_list = command_list + ["--skip-pull-image"] + + process = Popen(command_list, stdout=PIPE) + return_code = process.wait() + + self.assertEquals(return_code, 0) + process_stdout = b"".join(process.stdout.readlines()).strip() + self.assertEquals(process_stdout.decode('utf-8'), u'{"body":"hello 曰有冥 world 🐿","statusCode":200,"headers":{}}') diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index b749ef92d8..88dbea27dd 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -1,13 +1,24 @@ import json +import shutil +import os from nose_parameterized import parameterized from subprocess import Popen, PIPE from timeit import default_timer as timer +import docker + +from tests.integration.local.invoke.layer_utils import LayerUtils from .invoke_integ_base import InvokeIntegBase +try: + from pathlib import Path +except ImportError: + from pathlib2 import Path + class TestSamPython36HelloWorldIntegration(InvokeIntegBase): + template = Path("template.yml") def test_invoke_returncode_is_zero(self): command_list = self.get_command_list("HelloWorldServerlessFunction", @@ -175,3 +186,205 @@ def test_invoke_with_docker_network_of_host(self): return_code = process.wait() self.assertEquals(return_code, 0) + + +class TestLayerVersion(InvokeIntegBase): + template = Path("layers", "layer-template.yml") + region = 'us-west-2' + layer_utils = LayerUtils(region=region) + + def setUp(self): + self.layer_cache = Path().home().joinpath("integ_layer_cache") + self.layer_cache.mkdir(parents=True, exist_ok=True) + + def tearDown(self): + docker_client = docker.from_env() + samcli_images = docker_client.images.list(name='samcli/lambda') + for image in samcli_images: + docker_client.images.remove(image.id) + + shutil.rmtree(str(self.layer_cache)) + + @classmethod + def setUpClass(cls): + cls.layer_utils.upsert_layer(LayerUtils.generate_layer_name(), "LayerOneArn", "layer1.zip") + cls.layer_utils.upsert_layer(LayerUtils.generate_layer_name(), "LayerTwoArn", "layer2.zip") + super(TestLayerVersion, cls).setUpClass() + + @classmethod + def tearDownClass(cls): + cls.layer_utils.delete_layers() + super(TestLayerVersion, cls).tearDownClass() + + @parameterized.expand([ + ("ReferenceServerlessLayerVersionServerlessFunction"), + ("ReferenceLambdaLayerVersionServerlessFunction"), + ("ReferenceServerlessLayerVersionLambdaFunction"), + ("ReferenceLambdaLayerVersionLambdaFunction"), + ("ReferenceServerlessLayerVersionServerlessFunction") + ]) + def test_reference_of_layer_version(self, function_logical_id): + command_list = self.get_command_list(function_logical_id, + template_path=self.template_path, + no_event=True, + region=self.region, + layer_cache=str(self.layer_cache), + parameter_overrides=self.layer_utils.parameters_overrides + ) + + process = Popen(command_list, stdout=PIPE) + process.wait() + + process_stdout = b"".join(process.stdout.readlines()).strip() + + expected_output = '"This is a Layer Ping from simple_python"' + + self.assertEquals(process_stdout.decode('utf-8'), expected_output) + + @parameterized.expand([ + ("OneLayerVersionServerlessFunction"), + ("OneLayerVersionLambdaFunction") + ]) + def test_download_one_layer(self, function_logical_id): + command_list = self.get_command_list(function_logical_id, + template_path=self.template_path, + no_event=True, + region=self.region, + layer_cache=str(self.layer_cache), + parameter_overrides=self.layer_utils.parameters_overrides + ) + + process = Popen(command_list, stdout=PIPE) + process.wait() + + process_stdout = b"".join(process.stdout.readlines()[-1:]).strip() + expected_output = '"Layer1"' + + self.assertEquals(process_stdout.decode('utf-8'), expected_output) + + @parameterized.expand([ + ("ChangedLayerVersionServerlessFunction"), + ("ChangedLayerVersionLambdaFunction") + ]) + def test_publish_changed_download_layer(self, function_logical_id): + layer_name = self.layer_utils.generate_layer_name() + self.layer_utils.upsert_layer(layer_name=layer_name, + ref_layer_name="ChangedLayerArn", + layer_zip="layer1.zip") + + command_list = self.get_command_list(function_logical_id, + template_path=self.template_path, + no_event=True, + region=self.region, + layer_cache=str(self.layer_cache), + parameter_overrides=self.layer_utils.parameters_overrides + ) + + process = Popen(command_list, stdout=PIPE) + process.wait() + + process_stdout = b"".join(process.stdout.readlines()[-1:]).strip() + expected_output = '"Layer1"' + + self.assertEquals(process_stdout.decode('utf-8'), expected_output) + + self.layer_utils.upsert_layer(layer_name=layer_name, + ref_layer_name="ChangedLayerArn", + layer_zip="changedlayer1.zip") + + command_list = self.get_command_list(function_logical_id, + template_path=self.template_path, + no_event=True, + region=self.region, + layer_cache=str(self.layer_cache), + parameter_overrides=self.layer_utils.parameters_overrides + ) + + process = Popen(command_list, stdout=PIPE) + process.wait() + + process_stdout = b"".join(process.stdout.readlines()[-1:]).strip() + expected_output = '"Changed_Layer_1"' + + self.assertEquals(process_stdout.decode('utf-8'), expected_output) + + @parameterized.expand([ + ("TwoLayerVersionServerlessFunction"), + ("TwoLayerVersionLambdaFunction") + ]) + def test_download_two_layers(self, function_logical_id): + + command_list = self.get_command_list(function_logical_id, + template_path=self.template_path, + no_event=True, + region=self.region, + layer_cache=str(self.layer_cache), + parameter_overrides=self.layer_utils.parameters_overrides + ) + + process = Popen(command_list, stdout=PIPE) + process.wait() + + stdout = process.stdout.readlines() + + process_stdout = b"".join(stdout[-1:]).strip() + expected_output = '"Layer2"' + + self.assertEquals(process_stdout.decode('utf-8'), expected_output) + + def test_caching_two_layers(self): + + command_list = self.get_command_list("TwoLayerVersionServerlessFunction", + template_path=self.template_path, + no_event=True, + region=self.region, + layer_cache=str(self.layer_cache), + parameter_overrides=self.layer_utils.parameters_overrides + ) + + process = Popen(command_list, stdout=PIPE) + process.wait() + + self.assertEquals(2, len(os.listdir(str(self.layer_cache)))) + + def test_layer_does_not_exist(self): + + non_existent_layer_arn = self.layer_utils.parameters_overrides["LayerOneArn"].replace( + self.layer_utils.layers_meta[0].layer_name, 'non_existent_layer') + + command_list = self.get_command_list("LayerVersionDoesNotExistFunction", + template_path=self.template_path, + no_event=True, + region=self.region, + parameter_overrides={ + 'NonExistentLayerArn': non_existent_layer_arn + } + ) + + process = Popen(command_list, stderr=PIPE) + process.wait() + + process_stderr = b"".join(process.stderr.readlines()).strip() + error_output = process_stderr.decode('utf-8') + + expected_error_output = "{} was not found.".format(non_existent_layer_arn) + + self.assertIn(expected_error_output, error_output) + + def test_account_does_not_exist_for_layer(self): + command_list = self.get_command_list("LayerVersionAccountDoesNotExistFunction", + template_path=self.template_path, + no_event=True, + region=self.region + ) + + process = Popen(command_list, stderr=PIPE) + process.wait() + + process_stderr = b"".join(process.stderr.readlines()).strip() + error_output = process_stderr.decode('utf-8') + + expected_error_output = "Credentials provided are missing lambda:Getlayerversion policy that is needed to " \ + "download the layer or you do not have permission to download the layer" + + self.assertIn(expected_error_output, error_output) diff --git a/tests/integration/testdata/__init__.py b/tests/integration/testdata/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/buildcmd/template.yaml b/tests/integration/testdata/buildcmd/template.yaml index 716574ceb2..3f0bf45776 100644 --- a/tests/integration/testdata/buildcmd/template.yaml +++ b/tests/integration/testdata/buildcmd/template.yaml @@ -15,4 +15,7 @@ Resources: CodeUri: Python Timeout: 600 - + OtherRelativePathResource: + Type: AWS::ApiGateway::RestApi + Properties: + BodyS3Location: SomeRelativePath diff --git a/tests/integration/testdata/invoke/__init__.py b/tests/integration/testdata/invoke/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/invoke/layer_zips/changedlayer1.zip b/tests/integration/testdata/invoke/layer_zips/changedlayer1.zip new file mode 100644 index 0000000000000000000000000000000000000000..42a4ebb923b9fd5026e1c5d8b28f127c956fb355 GIT binary patch literal 576 zcmWIWW@h1H0D;p}QhdP-C?U!qz)+l-Tac3)UrIDl|#Mkb1B zs<=!;aV6Bq2|u4kF9RA2a-g@TZ?MMs6F%oU&U>!$(Y480|3(pr5XSL literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/invoke/layer_zips/layer1.zip b/tests/integration/testdata/invoke/layer_zips/layer1.zip new file mode 100644 index 0000000000000000000000000000000000000000..c628f71d5b13ebe243381134e1dd75737e3c9d3c GIT binary patch literal 1963 zcmWIWW@Zs#U|`^2m|YO(`-)MBX%!0t1B(I!11EzFgPu!pd~ivAQEF%iCj)co@f*>Z zKwMhE&A`agrR*usa;BQO@a@cKADK{zX-l_U zI#IM-XS21k&z!k8-<^5qq{)`NC97xI)g$fKrXMq3rKYBK?8lR>p1q#RAH}{p#@Cog z`PN)AHcsbs&Di8`TK4bcoBDG##n0~475kmjVMxdi%45@DX5foauTF?AeDh;^&6&?H z8cpmQ*tl90`MeWy|LgeAv#I>_B&pEX}0z(GuUUY-_@t{E_vQ= zP5wl=1i6%budFuxt~paYv-k|>XNJ!TpF^+TT+*8Jxp7TOinWN!8Rj$1k)}5%FR%9c zc5knQPk^G^_qVgZzvoV9@C!D4Wu)>pLM8W@@+`rHm3eNDW@SbB`o=HTS2xeUp4a!> z(r&)J^`?X+?m8#EX6+41y5%=_;+j?tYktYLZH0QS$3j$P)sEJ-<$11Jr!wcvsx=p) z(~gD4nqAuK7Bv0rOJBz23k)>lUpw9YuB1rpMy(SO1x`hLWOD;>K|<{(OkCX zy}qt;($0#6YR-D~fK6T*}EZ5<5@*LCPRX}hP?g-llJdKQ-@Ze?k7 zq<{IHdnNj1oX2-Mt$D1z$W6!XcCwW!d$mj9HNBrxQlbtQx-Q?k%duO1Y3gtGjJ0ZS zBl>l2mp)q6WU6QF#~;Ev`;LgexS^obi%DNO&IG9BD8#i)e~`L^DgQue1-o6t`3G!w zTFyTdt6{f4sQ*Fw4`2O(`5(-SBAS*)v?+xpF`Fd`mpw9kb13i4QAL~9{znFFA9T(= zlJThFpe=z-HWesEe!PPt3@*hS2Fx%HI|H%D^VO>-FgXur^MmxqoleMV* zt=(Xs(rG-Kt!DG>hi~L7ejJHC^z-)KZvx_Ccf(_X0z-rEU%Gd7v*DR&w{qh&zHmJY zz5vODIS=>thRiP8_^0lmy;pv)*!E5HD%nru%{Iu3UGn7z^FOb?O)e3eZq;`zJ|H~* z^BVT-zf30==6v?sv@r0)WScI-=PLbMxR_2ZowNI5pP$rS*UqxVcaLX^yfL3xE~gvv z`{k1vwsWs1@ZM|p*POonf%Xr^Acy^;5}!3-!N3M}l3Gjj`aQsWCM zOEU8F;&by;N^?^6!3Ce}+WXPxff%je0~rS?J3+=p`n|PQ1sbOg#HzTAL$O1zpb~C8 z$O<&$y*+({HO`;#IoEOCbB&Mgr3+gg1u--&x-#j~A{$m!`3Vaa3W+lyytE-i1I0@U zxZDsQpP83g5+4t<{x-*yXs?P3=+*~#Gcw6B{85_fEdjPbmNjn8&(`fqu2)y7<^Vie1RF@FkgVI fI)KY6l;8*20t$YNuwrEc1u6>=Rs&rw1?B+&#xxB~ literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/invoke/layer_zips/layer2.zip b/tests/integration/testdata/invoke/layer_zips/layer2.zip new file mode 100644 index 0000000000000000000000000000000000000000..df363b9c73db709362ca84801e58ff8db60f5ead GIT binary patch literal 1963 zcmWIWW@Zs#U|`^2m|YO(`-)MBX%!0t1B(I!11EzFgPu!pd~ivAQEF%iCj)co@f*>s z>+VOFR&X;gvUDkX%CnrQW-fd?GulTcRO0yl{Km4hs8fs!4Q?!z<`C2T9;C;6t?V;r?#*{+-Z^QqC2z^q6^>rm|k<{ z^NU6k`vx|y7DYbqgxvo+{_|`qKRr2lIaX}fv9}xEReyT&Qf`{9JAXvx zw_B4xQ7%C)W#22SO}}f-6wfR^!}*!vv%=@l>o=FQ=6r5klagXBqH>1$Omn2^&B@EF zy}sSsE8!EM==S~X?C`arpuR&G@*ycP}zlb8p$UWz7}VX|redUvf#mJ#nh&Jngl4 z@8srm>&}hex8c(h(d|oT&Sd7=R&wxa&MpVD#ZfJ0Oq)(FnYq$#`~8~rslVRN?yVF4 z*3qoq-eREXb@8143sZR%`r>)ooQ4#{qbCntB!$l zwkg+gw*EUM^E_;o+nT4HM_PSu34Hh3d*OueWPV#m!@_kPx=z~eDRm)}mAanAWrn>}N# z+S`bJo!h04RyCRGS^M#au+F|C;xBF}DD`5}SB^6QDme;qEz=*Q?qJG4P+Gxm*Kqy; z+ntv455;QO?GNgIkp9D0e_;Lx^P-5Rr4emPVM)wpNy2514Bs5edvjFLrnUc(LE8tN zbB|=aEcy#=+SQ^e80Q~M|5#bW7=Li}j<)EUEOSWCfcpsIE^n{ z&w?*NGGWfcy}co`%QpV0J818fA1tj}K~+Wj@BZ-1cuL$Izb{-M5Cikw)0lB-Qa-I@c=5zSjGBwlaiy)8ML zxq^xPqsmj`mSl^@$ri1FAsTbJS0puEnt8}X?@{m%lhiLtcJ0$Y7(M+GS|hvu;Vm&O z_U!?cw$fU`%AB#rUn+0hxKXuhb5V@;=I{TRt?U1P2j=tt5i@SvU%Jn2eSY@0IBSRg zojzRr@2Qm&)c7l!ju-Z};Xq-9_tKu>a#SXoKO0eux8bM5?nt~NlQ=run0p1|vaD_h1I01&Y zj&Fe#5K1^9;sR(iC@!!XjVmS~Ml%B4xTMjB6^GF%_JIQiXcZ`4uv!K21!jQ5d;zlR e04}Rgf*)uLDEKjKVPyjaDhm)+16?i!<^cdCHx2aw literal 0 HcmV?d00001 diff --git a/tests/integration/testdata/invoke/layers/__init__.py b/tests/integration/testdata/invoke/layers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/invoke/layers/custom_layer/__init__.py b/tests/integration/testdata/invoke/layers/custom_layer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/invoke/layers/custom_layer/my_layer/__init__.py b/tests/integration/testdata/invoke/layers/custom_layer/my_layer/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/invoke/layers/custom_layer/my_layer/simple_python.py b/tests/integration/testdata/invoke/layers/custom_layer/my_layer/simple_python.py new file mode 100644 index 0000000000..1998a450d7 --- /dev/null +++ b/tests/integration/testdata/invoke/layers/custom_layer/my_layer/simple_python.py @@ -0,0 +1,2 @@ +def layer_ping(): + return "This is a Layer Ping from simple_python" diff --git a/tests/integration/testdata/invoke/layers/layer-main.py b/tests/integration/testdata/invoke/layers/layer-main.py new file mode 100644 index 0000000000..44eed96c84 --- /dev/null +++ b/tests/integration/testdata/invoke/layers/layer-main.py @@ -0,0 +1,21 @@ +import sys +import site + +sys.path.insert(0, '/opt') +site.addsitedir("/opt") + + +def handler(event, context): + return 'hello' + + +def custom_layer_handler(event, context): + from my_layer.simple_python import layer_ping + + return layer_ping() + + +def one_layer_hanlder(event, context): + from simple_python_module.simple_python import which_layer + + return which_layer() diff --git a/tests/integration/testdata/invoke/layers/layer-template.yml b/tests/integration/testdata/invoke/layers/layer-template.yml new file mode 100644 index 0000000000..e494d42c6f --- /dev/null +++ b/tests/integration/testdata/invoke/layers/layer-template.yml @@ -0,0 +1,146 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: A hello world application. + +Parameters: + LayerOneArn: + Default: arn:aws:lambda:us-west-2:111111111111:layer:layer:1 + Type: String + + LayerTwoArn: + Default: arn:aws:lambda:us-west-2:111111111111:layer:layer2:1 + Type: String + + ChangedLayerArn: + Default: arn:aws:lambda:us-west-2:111111111111:layer:changed_layer:1 + Type: String + + NonExistentLayerArn: + Default: arn:aws:lambda:us-west-2:111111111111:layer:non_existent_layer:1 + Type: String + +Resources: + # AWS::Serverless::Function + OneLayerVersionServerlessFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.one_layer_hanlder + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: LayerOneArn + + # AWS::Serverless::Function + ChangedLayerVersionServerlessFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.one_layer_hanlder + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: ChangedLayerArn + + ReferenceServerlessLayerVersionServerlessFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.custom_layer_handler + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: MyCustomServerlessLayer + + ReferenceLambdaLayerVersionServerlessFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.custom_layer_handler + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: MyCustomLambdaLayer + + TwoLayerVersionServerlessFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.one_layer_hanlder + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: LayerOneArn + - Ref: LayerTwoArn + + # AWS::Lambda::Function + OneLayerVersionLambdaFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.one_layer_hanlder + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: LayerOneArn + + ChangedLayerVersionLambdaFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.one_layer_hanlder + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: ChangedLayerArn +# + ReferenceServerlessLayerVersionLambdaFunction: + Type: AWS::Lambda::Function + Properties: + Handler: layer-main.custom_layer_handler + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: MyCustomServerlessLayer + + ReferenceLambdaLayerVersionLambdaFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.custom_layer_handler + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: MyCustomLambdaLayer + + TwoLayerVersionLambdaFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.one_layer_hanlder + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: LayerOneArn + - Ref: LayerTwoArn + + LayerVersionDoesNotExistFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.handler + Runtime: python3.6 + CodeUri: . + Layers: + - Ref: NonExistentLayerArn + + LayerVersionAccountDoesNotExistFunction: + Type: AWS::Serverless::Function + Properties: + Handler: layer-main.handler + Runtime: python3.6 + CodeUri: . + Layers: + - arn:aws:lambda:us-west-2:111111111101:layer:layerDoesNotExist:1 + + # AWS::Lambda::LayerVersion + MyCustomLambdaLayer: + Type: AWS::Lambda::LayerVersion + Properties: + Content: custom_layer/ + + # AWS::Serverless::LayerVersion + MyCustomServerlessLayer: + Type: AWS::Serverless::LayerVersion + Properties: + ContentUri: custom_layer/ diff --git a/tests/integration/testdata/invoke/runtimes/custom_bash/bootstrap b/tests/integration/testdata/invoke/runtimes/custom_bash/bootstrap new file mode 100755 index 0000000000..e4a4a3aae9 --- /dev/null +++ b/tests/integration/testdata/invoke/runtimes/custom_bash/bootstrap @@ -0,0 +1,32 @@ +#!/bin/sh + +set -euo pipefail + +# Uncomment the next line to log the executed commands +# set -x + +# echo "Starting sample bash runtime." + +# Load the script that contains the handler function. +# +# Handler format: . +# The script file .sh must be located in +# the same directory as the bootstrap executable. +source $(dirname "$0")/"$(echo $_HANDLER | cut -d. -f1).sh" + +# Enter request processing loop +while true +do + # Request the next event from the Lambda Runtime + HEADERS="$(mktemp)" + EVENT_DATA=$(curl -sS -LD "$HEADERS" -X GET "http://${AWS_LAMBDA_RUNTIME_API}/2018-06-01/runtime/invocation/next") + INVOCATION_ID=$(grep -Fi Lambda-Runtime-Aws-Request-Id "$HEADERS" | tr -d '[:space:]' | cut -d: -f2) + +# echo "Invoke received. Request ID: $INVOCATION_ID" + + # Execute the handler function from the script + RESPONSE=$($(echo "$_HANDLER" | cut -d. -f2) "$EVENT_DATA") + + # Send the response to Lambda Runtime + IGNORE=$(curl -sS -X POST "http://${AWS_LAMBDA_RUNTIME_API}/2018-06-01/runtime/invocation/$INVOCATION_ID/response" -d "$RESPONSE") +done \ No newline at end of file diff --git a/tests/integration/testdata/invoke/runtimes/custom_bash/hello.sh b/tests/integration/testdata/invoke/runtimes/custom_bash/hello.sh new file mode 100755 index 0000000000..11d5fe13c2 --- /dev/null +++ b/tests/integration/testdata/invoke/runtimes/custom_bash/hello.sh @@ -0,0 +1,7 @@ +function echo_request () { + EVENT_DATA=$1 + + RESPONSE="{\"body\": \"hello 曰有冥 world 🐿\", \"statusCode\": 200, \"headers\": {}}" + + echo $RESPONSE +} \ No newline at end of file diff --git a/tests/integration/testdata/invoke/runtimes/template.yaml b/tests/integration/testdata/invoke/runtimes/template.yaml index 10012415f4..329ecb5e54 100644 --- a/tests/integration/testdata/invoke/runtimes/template.yaml +++ b/tests/integration/testdata/invoke/runtimes/template.yaml @@ -17,3 +17,17 @@ Resources: Runtime: java8 CodeUri: ./java8/target/HelloWorld-1.0.jar Timeout: 300 + + CustomBashFunction: + Type: AWS::Serverless::Function + Properties: + Handler: hello.echo_request + Runtime: provided + CodeUri: ./custom_bash + Timeout: 300 + Events: + MyApi: + Type: Api + Properties: + Path: / + Method: GET diff --git a/tests/integration/testdata/invoke/template.yml b/tests/integration/testdata/invoke/template.yml index 46a0c972e5..8a6e799f2f 100644 --- a/tests/integration/testdata/invoke/template.yml +++ b/tests/integration/testdata/invoke/template.yml @@ -24,7 +24,6 @@ Resources: Properties: Handler: main.handler Runtime: python3.6 - CodeUri: . TimeoutFunction: Type: AWS::Serverless::Function @@ -106,3 +105,4 @@ Resources: Timeout: !Ref DefaultTimeout MyRuntimeVersion: !Ref MyRuntimeVersion + diff --git a/tests/unit/commands/_utils/test_template.py b/tests/unit/commands/_utils/test_template.py index 305bf468cc..fb9876008c 100644 --- a/tests/unit/commands/_utils/test_template.py +++ b/tests/unit/commands/_utils/test_template.py @@ -1,14 +1,17 @@ +import os +import copy import yaml from unittest import TestCase from mock import patch, mock_open from parameterized import parameterized, param -from samcli.commands._utils.template import get_template_data +from samcli.commands._utils.template import get_template_data, _RESOURCES_WITH_LOCAL_PATHS, _update_relative_paths, \ + move_template -class TestInvokeContext_get_template_data(TestCase): +class Test_get_template_data(TestCase): def test_must_raise_if_file_does_not_exist(self): filename = "filename" @@ -61,3 +64,137 @@ def test_must_raise_on_parse_errors(self, exception, pathlib_mock, yaml_parse_mo actual_exception = ex_ctx.exception self.assertTrue(str(actual_exception).startswith("Failed to parse template: ")) + + +class Test_update_relative_paths(TestCase): + + def setUp(self): + + self.s3path = "s3://foo/bar" + self.abspath = os.path.abspath("tosomefolder") + self.curpath = os.path.join("foo", "bar") + self.src = os.path.abspath("src") # /path/from/root/src + self.dest = os.path.abspath(os.path.join("src", "destination")) # /path/from/root/src/destination + + self.expected_result = os.path.join("..", "foo", "bar") + + @parameterized.expand( + [(resource_type, props) for resource_type, props in _RESOURCES_WITH_LOCAL_PATHS.items()] + ) + def test_must_update_relative_paths(self, resource_type, properties): + + for propname in properties: + + template_dict = { + "Resources": { + "MyResourceWithRelativePath": { + "Type": resource_type, + "Properties": { + propname: self.curpath + } + }, + "MyResourceWithS3Path": { + "Type": resource_type, + "Properties": { + propname: self.s3path + } + }, + "MyResourceWithAbsolutePath": { + "Type": resource_type, + "Properties": { + propname: self.abspath + } + }, + "MyResourceWithInvalidPath": { + "Type": resource_type, + "Properties": { + # Path is not a string + propname: {"foo": "bar"} + } + }, + "MyResourceWithoutProperties": { + "Type": resource_type + }, + "UnsupportedResourceType": { + "Type": "AWS::Ec2::Instance", + "Properties": { + "Code": "bar" + } + }, + "ResourceWithoutType": {"foo": "bar"}, + }, + "Parameters": { + "a": "b" + } + } + + expected_template_dict = copy.deepcopy(template_dict) + expected_template_dict["Resources"]["MyResourceWithRelativePath"]["Properties"][propname] = \ + self.expected_result + + result = _update_relative_paths(template_dict, self.src, self.dest) + + self.maxDiff = None + self.assertEquals(result, expected_template_dict) + + def test_must_update_aws_include_also(self): + template_dict = { + "Resources": {"Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": self.curpath}}}, + "list_prop": [ + "a", + 1, 2, 3, + {"Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": self.curpath}}}, + + # S3 path + {"Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": self.s3path}}}, + ], + "Fn::Transform": {"Name": "AWS::OtherTransform"}, + "key1": {"Fn::Transform": "Invalid value"}, + "key2": {"Fn::Transform": {"no": "name"}} + } + + expected_template_dict = { + "Resources": {"Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": self.expected_result}}}, + "list_prop": [ + "a", + 1, 2, 3, + {"Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": self.expected_result}}}, + # S3 path + {"Fn::Transform": {"Name": "AWS::Include", "Parameters": {"Location": self.s3path}}}, + ], + "Fn::Transform": {"Name": "AWS::OtherTransform"}, + "key1": {"Fn::Transform": "Invalid value"}, + "key2": {"Fn::Transform": {"no": "name"}} + } + + result = _update_relative_paths(template_dict, self.src, self.dest) + self.maxDiff = None + self.assertEquals(result, expected_template_dict) + + +class Test_move_template(TestCase): + + @patch("samcli.commands._utils.template._update_relative_paths") + @patch("samcli.commands._utils.template.yaml_dump") + def test_must_update_and_write_template(self, + yaml_dump_mock, + update_relative_paths_mock): + template_dict = {"a": "b"} + + # Moving from /tmp/original/root/template.yaml to /tmp/new/root/othertemplate.yaml + source = os.path.join("/", "tmp", "original", "root", "template.yaml") + dest = os.path.join("/", "tmp", "new", "root", "othertemplate.yaml") + + modified_template = update_relative_paths_mock.return_value = "modified template" + dumped_yaml = yaml_dump_mock.return_value = "dump result" + + m = mock_open() + with patch("samcli.commands._utils.template.open", m): + move_template(source, dest, template_dict) + + update_relative_paths_mock.assert_called_once_with(template_dict, + os.path.dirname(source), + os.path.dirname(dest)) + yaml_dump_mock.assert_called_with(modified_template) + m.assert_called_with(dest, 'w') + m.return_value.write.assert_called_with(dumped_yaml) diff --git a/tests/unit/commands/buildcmd/test_command.py b/tests/unit/commands/buildcmd/test_command.py index df8bf83a55..3c6700fb6d 100644 --- a/tests/unit/commands/buildcmd/test_command.py +++ b/tests/unit/commands/buildcmd/test_command.py @@ -1,6 +1,6 @@ from unittest import TestCase -from mock import Mock, patch, mock_open +from mock import Mock, patch from parameterized import parameterized from samcli.commands.build.command import do_cli @@ -12,9 +12,13 @@ class TestDoCli(TestCase): @patch("samcli.commands.build.command.BuildContext") @patch("samcli.commands.build.command.ApplicationBuilder") - @patch("samcli.commands.build.command.yaml_dump") + @patch("samcli.commands.build.command.move_template") @patch("samcli.commands.build.command.os") - def test_must_succeed_build(self, os_mock, yaml_dump_mock, ApplicationBuilderMock, BuildContextMock): + def test_must_succeed_build(self, + os_mock, + move_template_mock, + ApplicationBuilderMock, + BuildContextMock): ctx_mock = Mock() BuildContextMock.return_value.__enter__ = Mock() @@ -22,12 +26,9 @@ def test_must_succeed_build(self, os_mock, yaml_dump_mock, ApplicationBuilderMoc builder_mock = ApplicationBuilderMock.return_value = Mock() artifacts = builder_mock.build.return_value = "artifacts" modified_template = builder_mock.update_template.return_value = "modified template" - dumped_yaml = yaml_dump_mock.return_value = "dumped yaml" - m = mock_open() - with patch("samcli.commands.build.command.open", m): - do_cli("template", "base_dir", "build_dir", "clean", "use_container", - "manifest_path", "docker_network", "skip_pull", "parameter_overrides") + do_cli("template", "base_dir", "build_dir", "clean", "use_container", + "manifest_path", "docker_network", "skip_pull", "parameter_overrides") ApplicationBuilderMock.assert_called_once_with(ctx_mock.function_provider, ctx_mock.build_dir, @@ -36,12 +37,11 @@ def test_must_succeed_build(self, os_mock, yaml_dump_mock, ApplicationBuilderMoc container_manager=ctx_mock.container_manager) builder_mock.build.assert_called_once() builder_mock.update_template.assert_called_once_with(ctx_mock.template_dict, - ctx_mock.output_template_path, + ctx_mock.original_template_path, artifacts) - - yaml_dump_mock.assert_called_with(modified_template) - m.assert_called_with(ctx_mock.output_template_path, 'w') - m.return_value.write.assert_called_with(dumped_yaml) + move_template_mock.assert_called_once_with(ctx_mock.original_template_path, + ctx_mock.output_template_path, + modified_template) @parameterized.expand([ (UnsupportedRuntimeException(), ), diff --git a/tests/unit/commands/local/cli_common/test_invoke_context.py b/tests/unit/commands/local/cli_common/test_invoke_context.py index 9e842dc8ad..b78b6abfff 100644 --- a/tests/unit/commands/local/cli_common/test_invoke_context.py +++ b/tests/unit/commands/local/cli_common/test_invoke_context.py @@ -35,12 +35,11 @@ def test_must_read_from_necessary_files(self, SamFunctionProviderMock): docker_network="network", log_file=log_file, skip_pull_image=True, - aws_profile="profile", debug_port=1111, debugger_path="path-to-debugger", debug_args='args', - aws_region="region", - parameter_overrides={}) + parameter_overrides={}, + aws_region="region") template_dict = "template_dict" invoke_context._get_template_data = Mock() @@ -117,11 +116,9 @@ def test_must_work_in_with_statement(self, ExitMock, EnterMock): docker_network="network", log_file="log_file", skip_pull_image=True, - aws_profile="profile", debug_port=1111, debugger_path="path-to-debugger", - debug_args='args', - aws_region="region") as context: + debug_args='args') as context: self.assertEquals(context_obj, context) EnterMock.assert_called_with() @@ -166,16 +163,22 @@ def setUp(self): docker_network="network", log_file="log_file", skip_pull_image=True, - aws_profile="profile", + force_image_build=True, debug_port=1111, debugger_path="path-to-debugger", - debug_args='args', - aws_region="region") + debug_args='args') + @patch("samcli.commands.local.cli_common.invoke_context.LambdaImage") + @patch("samcli.commands.local.cli_common.invoke_context.LayerDownloader") @patch("samcli.commands.local.cli_common.invoke_context.ContainerManager") @patch("samcli.commands.local.cli_common.invoke_context.LambdaRuntime") @patch("samcli.commands.local.cli_common.invoke_context.LocalLambdaRunner") - def test_must_create_runner(self, LocalLambdaMock, LambdaRuntimeMock, ContainerManagerMock): + def test_must_create_runner(self, + LocalLambdaMock, + LambdaRuntimeMock, + ContainerManagerMock, + download_layers_mock, + lambda_image_patch): container_mock = Mock() ContainerManagerMock.return_value = container_mock @@ -186,6 +189,12 @@ def test_must_create_runner(self, LocalLambdaMock, LambdaRuntimeMock, ContainerM runner_mock = Mock() LocalLambdaMock.return_value = runner_mock + download_mock = Mock() + download_layers_mock.return_value = download_mock + + image_mock = Mock() + lambda_image_patch.return_value = image_mock + cwd = "cwd" self.context.get_cwd = Mock() self.context.get_cwd.return_value = cwd @@ -195,14 +204,13 @@ def test_must_create_runner(self, LocalLambdaMock, LambdaRuntimeMock, ContainerM ContainerManagerMock.assert_called_with(docker_network_id="network", skip_pull_image=True) - LambdaRuntimeMock.assert_called_with(container_mock) + LambdaRuntimeMock.assert_called_with(container_mock, image_mock) + lambda_image_patch.assert_called_once_with(download_mock, True, True) LocalLambdaMock.assert_called_with(local_runtime=runtime_mock, function_provider=ANY, cwd=cwd, debug_context=None, - env_vars_values=ANY, - aws_profile="profile", - aws_region="region") + env_vars_values=ANY) class TestInvokeContext_stdout_property(TestCase): diff --git a/tests/unit/commands/local/invoke/test_cli.py b/tests/unit/commands/local/invoke/test_cli.py index d8129cbdfa..b4fdf459e8 100644 --- a/tests/unit/commands/local/invoke/test_cli.py +++ b/tests/unit/commands/local/invoke/test_cli.py @@ -7,11 +7,13 @@ from parameterized import parameterized, param from samcli.local.lambdafn.exceptions import FunctionNotFound +from samcli.commands.local.lib.exceptions import InvalidLayerReference from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.exceptions import UserException from samcli.commands.local.invoke.cli import do_cli as invoke_cli, _get_event as invoke_cli_get_event from samcli.commands.local.lib.exceptions import OverridesNotWellDefinedError from samcli.local.docker.manager import DockerImagePullFailedException +from samcli.local.docker.lambda_container import DebuggingNotSupported STDIN_FILE_NAME = "-" @@ -31,10 +33,11 @@ def setUp(self): self.docker_network = "network" self.log_file = "logfile" self.skip_pull_image = True - self.profile = "profile" self.no_event = False - self.region = "region" self.parameter_overrides = {} + self.layer_cache_basedir = "/some/layers/path" + self.force_image_build = True + self.region_name = "region" @patch("samcli.commands.local.invoke.cli.InvokeContext") @patch("samcli.commands.local.invoke.cli._get_event") @@ -42,11 +45,14 @@ def test_cli_must_setup_context_and_invoke(self, get_event_mock, InvokeContextMo event_data = "data" get_event_mock.return_value = event_data + ctx_mock = Mock() + ctx_mock.region = self.region_name + # Mock the __enter__ method to return a object inside a context manager context_mock = Mock() InvokeContextMock.return_value.__enter__.return_value = context_mock - invoke_cli(ctx=None, + invoke_cli(ctx=ctx_mock, function_identifier=self.function_id, template=self.template, event=self.eventfile, @@ -59,9 +65,9 @@ def test_cli_must_setup_context_and_invoke(self, get_event_mock, InvokeContextMo docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) InvokeContextMock.assert_called_with(template_file=self.template, function_identifier=self.function_id, @@ -70,12 +76,13 @@ def test_cli_must_setup_context_and_invoke(self, get_event_mock, InvokeContextMo docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - aws_profile=self.profile, debug_port=self.debug_port, debug_args=self.debug_args, debugger_path=self.debugger_path, - aws_region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build, + aws_region=self.region_name) context_mock.local_lambda_runner.invoke.assert_called_with(context_mock.function_name, event=event_data, @@ -87,10 +94,14 @@ def test_cli_must_setup_context_and_invoke(self, get_event_mock, InvokeContextMo @patch("samcli.commands.local.invoke.cli._get_event") def test_cli_must_invoke_with_no_event(self, get_event_mock, InvokeContextMock): self.no_event = True + + ctx_mock = Mock() + ctx_mock.region = self.region_name + # Mock the __enter__ method to return a object inside a context manager context_mock = Mock() InvokeContextMock.return_value.__enter__.return_value = context_mock - invoke_cli(ctx=None, + invoke_cli(ctx=ctx_mock, function_identifier=self.function_id, template=self.template, event=STDIN_FILE_NAME, @@ -103,9 +114,9 @@ def test_cli_must_invoke_with_no_event(self, get_event_mock, InvokeContextMock): docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) InvokeContextMock.assert_called_with(template_file=self.template, function_identifier=self.function_id, @@ -114,12 +125,13 @@ def test_cli_must_invoke_with_no_event(self, get_event_mock, InvokeContextMock): docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - aws_profile=self.profile, debug_port=self.debug_port, debug_args=self.debug_args, debugger_path=self.debugger_path, - aws_region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build, + aws_region=self.region_name) context_mock.local_lambda_runner.invoke.assert_called_with(context_mock.function_name, event="{}", @@ -132,9 +144,12 @@ def test_cli_must_invoke_with_no_event(self, get_event_mock, InvokeContextMock): def test_must_raise_user_exception_on_no_event_and_event(self, get_event_mock, InvokeContextMock): self.no_event = True + ctx_mock = Mock() + ctx_mock.region = self.region_name + with self.assertRaises(UserException) as ex_ctx: - invoke_cli(ctx=None, + invoke_cli(ctx=ctx_mock, function_identifier=self.function_id, template=self.template, event=self.eventfile, @@ -147,9 +162,9 @@ def test_must_raise_user_exception_on_no_event_and_event(self, get_event_mock, I docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) msg = str(ex_ctx.exception) self.assertEquals(msg, "no_event and event cannot be used together. Please provide only one.") @@ -168,6 +183,9 @@ def test_must_raise_user_exception_on_function_not_found(self, event_data = "data" get_event_mock.return_value = event_data + ctx_mock = Mock() + ctx_mock.region = self.region_name + # Mock the __enter__ method to return a object inside a context manager context_mock = Mock() InvokeContextMock.return_value.__enter__.return_value = context_mock @@ -176,7 +194,7 @@ def test_must_raise_user_exception_on_function_not_found(self, with self.assertRaises(UserException) as ex_ctx: - invoke_cli(ctx=None, + invoke_cli(ctx=ctx_mock, function_identifier=self.function_id, template=self.template, event=self.eventfile, @@ -189,24 +207,36 @@ def test_must_raise_user_exception_on_function_not_found(self, docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) msg = str(ex_ctx.exception) self.assertEquals(msg, expected_exectpion_message) + @parameterized.expand([(InvalidSamDocumentException("bad template"), "bad template"), + (InvalidLayerReference(), "Layer References need to be of type " + "'AWS::Serverless::LayerVersion' or 'AWS::Lambda::LayerVersion'"), + (DebuggingNotSupported("Debugging not supported"), "Debugging not supported") + ]) @patch("samcli.commands.local.invoke.cli.InvokeContext") @patch("samcli.commands.local.invoke.cli._get_event") - def test_must_raise_user_exception_on_invalid_sam_template(self, get_event_mock, InvokeContextMock): + def test_must_raise_user_exception_on_invalid_sam_template(self, + exeception_to_raise, + execption_message, + get_event_mock, + InvokeContextMock): event_data = "data" get_event_mock.return_value = event_data - InvokeContextMock.side_effect = InvalidSamDocumentException("bad template") + ctx_mock = Mock() + ctx_mock.region = self.region_name + + InvokeContextMock.side_effect = exeception_to_raise with self.assertRaises(UserException) as ex_ctx: - invoke_cli(ctx=None, + invoke_cli(ctx=ctx_mock, function_identifier=self.function_id, template=self.template, event=self.eventfile, @@ -219,12 +249,12 @@ def test_must_raise_user_exception_on_invalid_sam_template(self, get_event_mock, docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) msg = str(ex_ctx.exception) - self.assertEquals(msg, "bad template") + self.assertEquals(msg, execption_message) @patch("samcli.commands.local.invoke.cli.InvokeContext") @patch("samcli.commands.local.invoke.cli._get_event") @@ -232,11 +262,14 @@ def test_must_raise_user_exception_on_invalid_env_vars(self, get_event_mock, Inv event_data = "data" get_event_mock.return_value = event_data + ctx_mock = Mock() + ctx_mock.region = self.region_name + InvokeContextMock.side_effect = OverridesNotWellDefinedError("bad env vars") with self.assertRaises(UserException) as ex_ctx: - invoke_cli(ctx=None, + invoke_cli(ctx=ctx_mock, function_identifier=self.function_id, template=self.template, event=self.eventfile, @@ -249,9 +282,9 @@ def test_must_raise_user_exception_on_invalid_env_vars(self, get_event_mock, Inv docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) msg = str(ex_ctx.exception) self.assertEquals(msg, "bad env vars") diff --git a/tests/unit/commands/local/lib/test_local_lambda.py b/tests/unit/commands/local/lib/test_local_lambda.py index f2bbfed631..b0366aae97 100644 --- a/tests/unit/commands/local/lib/test_local_lambda.py +++ b/tests/unit/commands/local/lib/test_local_lambda.py @@ -1,8 +1,6 @@ """ Testing local lambda runner """ -import os - from unittest import TestCase from mock import Mock, patch from parameterized import parameterized, param @@ -33,9 +31,7 @@ def setUp(self): self.function_provider_mock, self.cwd, env_vars_values=self.env_vars_values, - debug_context=self.debug_context, - aws_profile=self.aws_profile, - aws_region=self.aws_region) + debug_context=self.debug_context) @patch("samcli.commands.local.lib.local_lambda.boto3") def test_must_get_from_boto_session(self, boto3_mock): @@ -47,7 +43,7 @@ def test_must_get_from_boto_session(self, boto3_mock): mock_session = Mock() mock_session.region_name = self.region - boto3_mock.session.Session.return_value = mock_session + boto3_mock.DEFAULT_SESSION = mock_session mock_session.get_credentials.return_value = creds expected = { @@ -60,8 +56,6 @@ def test_must_get_from_boto_session(self, boto3_mock): actual = self.local_lambda.get_aws_creds() self.assertEquals(expected, actual) - boto3_mock.session.Session.assert_called_with(profile_name=self.aws_profile, region_name=self.aws_region) - @patch("samcli.commands.local.lib.local_lambda.boto3") def test_must_work_with_no_region_name(self, boto3_mock): creds = Mock() @@ -72,6 +66,7 @@ def test_must_work_with_no_region_name(self, boto3_mock): mock_session = Mock() del mock_session.region_name # Ask mock to return AttributeError when 'region_name' is accessed + boto3_mock.DEFAULT_SESSION = None boto3_mock.session.Session.return_value = mock_session mock_session.get_credentials.return_value = creds @@ -84,7 +79,7 @@ def test_must_work_with_no_region_name(self, boto3_mock): actual = self.local_lambda.get_aws_creds() self.assertEquals(expected, actual) - boto3_mock.session.Session.assert_called_with(profile_name=self.aws_profile, region_name=self.aws_region) + boto3_mock.session.Session.assert_called_with() @patch("samcli.commands.local.lib.local_lambda.boto3") def test_must_work_with_no_access_key(self, boto3_mock): @@ -96,6 +91,7 @@ def test_must_work_with_no_access_key(self, boto3_mock): mock_session = Mock() mock_session.region_name = self.region + boto3_mock.DEFAULT_SESSION = None boto3_mock.session.Session.return_value = mock_session mock_session.get_credentials.return_value = creds @@ -108,7 +104,7 @@ def test_must_work_with_no_access_key(self, boto3_mock): actual = self.local_lambda.get_aws_creds() self.assertEquals(expected, actual) - boto3_mock.session.Session.assert_called_with(profile_name=self.aws_profile, region_name=self.aws_region) + boto3_mock.session.Session.assert_called_with() @patch("samcli.commands.local.lib.local_lambda.boto3") def test_must_work_with_no_secret_key(self, boto3_mock): @@ -120,6 +116,7 @@ def test_must_work_with_no_secret_key(self, boto3_mock): mock_session = Mock() mock_session.region_name = self.region + boto3_mock.DEFAULT_SESSION = None boto3_mock.session.Session.return_value = mock_session mock_session.get_credentials.return_value = creds @@ -132,7 +129,7 @@ def test_must_work_with_no_secret_key(self, boto3_mock): actual = self.local_lambda.get_aws_creds() self.assertEquals(expected, actual) - boto3_mock.session.Session.assert_called_with(profile_name=self.aws_profile, region_name=self.aws_region) + boto3_mock.session.Session.assert_called_with() @patch("samcli.commands.local.lib.local_lambda.boto3") def test_must_work_with_no_session_token(self, boto3_mock): @@ -144,6 +141,7 @@ def test_must_work_with_no_session_token(self, boto3_mock): mock_session = Mock() mock_session.region_name = self.region + boto3_mock.DEFAULT_SESSION = None boto3_mock.session.Session.return_value = mock_session mock_session.get_credentials.return_value = creds @@ -156,10 +154,11 @@ def test_must_work_with_no_session_token(self, boto3_mock): actual = self.local_lambda.get_aws_creds() self.assertEquals(expected, actual) - boto3_mock.session.Session.assert_called_with(profile_name=self.aws_profile, region_name=self.aws_region) + boto3_mock.session.Session.assert_called() @patch("samcli.commands.local.lib.local_lambda.boto3") def test_must_work_with_no_credentials(self, boto3_mock): + boto3_mock.DEFAULT_SESSION = None mock_session = Mock() boto3_mock.session.Session.return_value = mock_session mock_session.get_credentials.return_value = None @@ -168,98 +167,18 @@ def test_must_work_with_no_credentials(self, boto3_mock): actual = self.local_lambda.get_aws_creds() self.assertEquals(expected, actual) - boto3_mock.session.Session.assert_called_with(profile_name=self.aws_profile, region_name=self.aws_region) + boto3_mock.session.Session.assert_called() @patch("samcli.commands.local.lib.local_lambda.boto3") def test_must_work_with_no_session(self, boto3_mock): + boto3_mock.DEFAULT_SESSION = None boto3_mock.session.Session.return_value = None expected = {} actual = self.local_lambda.get_aws_creds() self.assertEquals(expected, actual) - boto3_mock.session.Session.assert_called_with(profile_name=self.aws_profile, region_name=self.aws_region) - - -class TestLocalLambda_get_code_path(TestCase): - - def setUp(self): - self.runtime_mock = Mock() - self.function_provider_mock = Mock() - self.cwd = "/my/current/working/directory" - self.env_vars_values = {} - self.debug_context = None - self.aws_profile = "myprofile" - self.aws_region = "region" - - self.relative_codeuri = "./my/path" - self.absolute_codeuri = "/home/foo/bar" # Some absolute path to use - self.os_cwd = os.getcwd() - - self.local_lambda = LocalLambdaRunner(self.runtime_mock, - self.function_provider_mock, - self.cwd, - env_vars_values=self.env_vars_values, - aws_profile=self.aws_profile, - debug_context=self.debug_context, - aws_region=self.aws_region) - - @parameterized.expand([ - ("."), - ("") - ]) - def test_must_resolve_present_cwd(self, cwd_path): - - self.local_lambda.cwd = cwd_path - codeuri = self.relative_codeuri - expected = os.path.normpath(os.path.join(self.os_cwd, codeuri)) - - actual = self.local_lambda._get_code_path(codeuri) - self.assertEquals(expected, actual) - self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") - - @parameterized.expand([ - ("var/task"), - ("some/dir") - ]) - def test_must_resolve_relative_cwd(self, cwd_path): - - self.local_lambda.cwd = cwd_path - codeuri = self.relative_codeuri - - abs_cwd = os.path.abspath(cwd_path) - expected = os.path.normpath(os.path.join(abs_cwd, codeuri)) - - actual = self.local_lambda._get_code_path(codeuri) - self.assertEquals(expected, actual) - self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") - - @parameterized.expand([ - (""), - ("."), - ("hello"), - ("a/b/c/d"), - ("../../c/d/e") - ]) - def test_must_resolve_relative_codeuri(self, codeuri): - - expected = os.path.normpath(os.path.join(self.cwd, codeuri)) - - actual = self.local_lambda._get_code_path(codeuri) - self.assertEquals(expected, actual) - self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") - - @parameterized.expand([ - ("/a/b/c"), - ("/") - ]) - def test_must_resolve_absolute_codeuri(self, codeuri): - - expected = codeuri # CodeUri must be return as is for absolute paths - - actual = self.local_lambda._get_code_path(codeuri) - self.assertEquals(expected, actual) - self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") + boto3_mock.session.Session.assert_called() class TestLocalLambda_make_env_vars(TestCase): @@ -283,9 +202,7 @@ def setUp(self): self.function_provider_mock, self.cwd, env_vars_values=self.env_vars_values, - debug_context=self.debug_context, - aws_profile=self.aws_profile, - aws_region=self.aws_region) + debug_context=self.debug_context) self.aws_creds = {"key": "key"} self.local_lambda.get_aws_creds = Mock() @@ -315,7 +232,8 @@ def test_must_work_with_override_values(self, env_vars_values, expected_override handler="handler", codeuri="codeuri", environment=self.environ, - rolearn=None) + rolearn=None, + layers=[]) self.local_lambda.env_vars_values = env_vars_values @@ -349,7 +267,8 @@ def test_must_not_work_with_invalid_override_values(self, env_vars_values, expec handler="handler", codeuri="codeuri", environment=self.environ, - rolearn=None) + rolearn=None, + layers=[]) self.local_lambda.env_vars_values = env_vars_values @@ -374,7 +293,8 @@ def test_must_work_with_invalid_environment_variable(self, environment_variable, handler="handler", codeuri="codeuri", environment=environment_variable, - rolearn=None) + rolearn=None, + layers=[]) self.local_lambda.env_vars_values = {} @@ -404,13 +324,12 @@ def setUp(self): self.function_provider_mock, self.cwd, env_vars_values=self.env_vars_values, - aws_profile=self.aws_profile, - debug_context=self.debug_context, - aws_region=self.aws_region) + debug_context=self.debug_context) + @patch('samcli.commands.local.lib.local_lambda.resolve_code_path') @patch('samcli.commands.local.lib.local_lambda.LocalLambdaRunner.is_debugging') @patch('samcli.commands.local.lib.local_lambda.FunctionConfig') - def test_must_work(self, FunctionConfigMock, is_debugging_mock): + def test_must_work(self, FunctionConfigMock, is_debugging_mock, resolve_code_path_patch): is_debugging_mock.return_value = False env_vars = "envvars" @@ -418,8 +337,9 @@ def test_must_work(self, FunctionConfigMock, is_debugging_mock): self.local_lambda._make_env_vars.return_value = env_vars codepath = "codepath" - self.local_lambda._get_code_path = Mock() - self.local_lambda._get_code_path.return_value = codepath + resolve_code_path_patch.return_value = codepath + + layers = ['layer1', 'layer2'] function = Function(name="function_name", runtime="runtime", @@ -428,7 +348,8 @@ def test_must_work(self, FunctionConfigMock, is_debugging_mock): handler="handler", codeuri="codeuri", environment=None, - rolearn=None) + rolearn=None, + layers=layers) config = "someconfig" FunctionConfigMock.return_value = config @@ -439,16 +360,18 @@ def test_must_work(self, FunctionConfigMock, is_debugging_mock): runtime=function.runtime, handler=function.handler, code_abs_path=codepath, + layers=layers, memory=function.memory, timeout=function.timeout, env_vars=env_vars) - self.local_lambda._get_code_path.assert_called_with(function.codeuri) + resolve_code_path_patch.assert_called_with(self.cwd, function.codeuri) self.local_lambda._make_env_vars.assert_called_with(function) + @patch('samcli.commands.local.lib.local_lambda.resolve_code_path') @patch('samcli.commands.local.lib.local_lambda.LocalLambdaRunner.is_debugging') @patch('samcli.commands.local.lib.local_lambda.FunctionConfig') - def test_timeout_set_to_max_during_debugging(self, FunctionConfigMock, is_debugging_mock): + def test_timeout_set_to_max_during_debugging(self, FunctionConfigMock, is_debugging_mock, resolve_code_path_patch): is_debugging_mock.return_value = True env_vars = "envvars" @@ -456,8 +379,7 @@ def test_timeout_set_to_max_during_debugging(self, FunctionConfigMock, is_debugg self.local_lambda._make_env_vars.return_value = env_vars codepath = "codepath" - self.local_lambda._get_code_path = Mock() - self.local_lambda._get_code_path.return_value = codepath + resolve_code_path_patch.return_value = codepath function = Function(name="function_name", runtime="runtime", @@ -466,7 +388,8 @@ def test_timeout_set_to_max_during_debugging(self, FunctionConfigMock, is_debugg handler="handler", codeuri="codeuri", environment=None, - rolearn=None) + rolearn=None, + layers=[]) config = "someconfig" FunctionConfigMock.return_value = config @@ -477,11 +400,12 @@ def test_timeout_set_to_max_during_debugging(self, FunctionConfigMock, is_debugg runtime=function.runtime, handler=function.handler, code_abs_path=codepath, + layers=[], memory=function.memory, timeout=function.timeout, env_vars=env_vars) - self.local_lambda._get_code_path.assert_called_with(function.codeuri) + resolve_code_path_patch.assert_called_with(self.cwd, function.codeuri) self.local_lambda._make_env_vars.assert_called_with(function) @@ -500,9 +424,7 @@ def setUp(self): self.function_provider_mock, self.cwd, env_vars_values=self.env_vars_values, - aws_profile=self.aws_profile, - debug_context=self.debug_context, - aws_region=self.aws_region) + debug_context=self.debug_context) def test_must_work(self): name = "name" @@ -544,9 +466,7 @@ def setUp(self): self.function_provider_mock, self.cwd, env_vars_values=self.env_vars_values, - aws_profile=self.aws_profile, - debug_context=self.debug_context, - aws_region=self.aws_region) + debug_context=self.debug_context) def test_must_be_on(self): self.assertTrue(self.local_lambda.is_debugging()) @@ -557,8 +477,6 @@ def test_must_be_off(self): self.function_provider_mock, self.cwd, env_vars_values=self.env_vars_values, - debug_context=None, - aws_profile=self.aws_profile, - aws_region=self.aws_region) + debug_context=None) self.assertFalse(self.local_lambda.is_debugging()) diff --git a/tests/unit/commands/local/lib/test_provider.py b/tests/unit/commands/local/lib/test_provider.py new file mode 100644 index 0000000000..20492868e9 --- /dev/null +++ b/tests/unit/commands/local/lib/test_provider.py @@ -0,0 +1,58 @@ +from unittest import TestCase + +from parameterized import parameterized + +from samcli.commands.local.lib.provider import LayerVersion +from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn, UnsupportedIntrinsic + + +class TestLayerVersion(TestCase): + + @parameterized.expand([ + ("arn:aws:lambda:region:account-id:layer:layer-name:a"), + ("arn:aws:lambda:region:account-id:layer"), + ("a string without delimiter") + ]) + def test_invalid_arn(self, arn): + with self.assertRaises(InvalidLayerVersionArn): + LayerVersion(arn, None) + + def test_layer_version_returned(self): + layer_version = LayerVersion("arn:aws:lambda:region:account-id:layer:layer-name:1", None) + + self.assertEquals(layer_version.version, 1) + + def test_layer_arn_returned(self): + layer_version = LayerVersion("arn:aws:lambda:region:account-id:layer:layer-name:1", None) + + self.assertEquals(layer_version.layer_arn, "arn:aws:lambda:region:account-id:layer:layer-name") + + def test_codeuri_is_setable(self): + layer_version = LayerVersion("arn:aws:lambda:region:account-id:layer:layer-name:1", None) + layer_version.codeuri = "./some_value" + + self.assertEquals(layer_version.codeuri, "./some_value") + + def test_name_is_computed(self): + layer_version = LayerVersion("arn:aws:lambda:region:account-id:layer:layer-name:1", None) + + self.assertEquals(layer_version.name, "layer-name-1-8cebcd0539") + + def test_layer_version_is_defined_in_template(self): + layer_version = LayerVersion("arn:aws:lambda:region:account-id:layer:layer-name:1", ".") + + self.assertTrue(layer_version.is_defined_within_template) + + def test_layer_version_raises_unsupported_intrinsic(self): + intrinsic_arn = { + "Fn::Sub": + [ + "arn:aws:lambda:region:account-id:layer:{layer_name}:1", + { + "layer_name": "layer-name" + } + ] + } + + with self.assertRaises(UnsupportedIntrinsic): + LayerVersion(intrinsic_arn, ".") diff --git a/tests/unit/commands/local/lib/test_sam_function_provider.py b/tests/unit/commands/local/lib/test_sam_function_provider.py index 3c16573bd1..5777972b64 100644 --- a/tests/unit/commands/local/lib/test_sam_function_provider.py +++ b/tests/unit/commands/local/lib/test_sam_function_provider.py @@ -2,8 +2,9 @@ from mock import patch from parameterized import parameterized -from samcli.commands.local.lib.provider import Function +from samcli.commands.local.lib.provider import Function, LayerVersion from samcli.commands.local.lib.sam_function_provider import SamFunctionProvider +from samcli.commands.local.lib.exceptions import InvalidLayerReference class TestSamFunctionProviderEndToEnd(TestCase): @@ -54,6 +55,14 @@ class TestSamFunctionProviderEndToEnd(TestCase): "Handler": "index.handler" } }, + "LambdaFuncWithLocalPath": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": "./some/path/to/code", + "Runtime": "nodejs4.3", + "Handler": "index.handler" + } + }, "OtherResource": { "Type": "AWS::Serverless::Api", "Properties": { @@ -79,7 +88,8 @@ def setUp(self): memory=None, timeout=None, environment=None, - rolearn=None + rolearn=None, + layers=[] )), ("SamFunc2", Function( name="SamFunc2", @@ -89,7 +99,8 @@ def setUp(self): memory=None, timeout=None, environment=None, - rolearn=None + rolearn=None, + layers=[] )), ("SamFunc3", Function( name="SamFunc3", @@ -99,7 +110,8 @@ def setUp(self): memory=None, timeout=None, environment=None, - rolearn=None + rolearn=None, + layers=[] )), ("LambdaFunc1", Function( name="LambdaFunc1", @@ -109,7 +121,19 @@ def setUp(self): memory=None, timeout=None, environment=None, - rolearn=None + rolearn=None, + layers=[] + )), + ("LambdaFuncWithLocalPath", Function( + name="LambdaFuncWithLocalPath", + runtime="nodejs4.3", + handler="index.handler", + codeuri="./some/path/to/code", + memory=None, + timeout=None, + environment=None, + rolearn=None, + layers=[] )) ]) def test_get_must_return_each_function(self, name, expected_output): @@ -120,7 +144,7 @@ def test_get_must_return_each_function(self, name, expected_output): def test_get_all_must_return_all_functions(self): result = {f.name for f in self.provider.get_all()} - expected = {"SamFunc1", "SamFunc2", "SamFunc3", "LambdaFunc1"} + expected = {"SamFunc1", "SamFunc2", "SamFunc3", "LambdaFunc1", "LambdaFuncWithLocalPath"} self.assertEquals(result, expected) @@ -179,7 +203,7 @@ def test_must_work_for_sam_function(self, convert_mock): result = SamFunctionProvider._extract_functions(resources) self.assertEquals(expected, result) - convert_mock.assert_called_with('Func1', {"a": "b"}) + convert_mock.assert_called_with('Func1', {"a": "b"}, []) @patch.object(SamFunctionProvider, "_convert_sam_function_resource") def test_must_work_with_no_properties(self, convert_mock): @@ -199,7 +223,7 @@ def test_must_work_with_no_properties(self, convert_mock): result = SamFunctionProvider._extract_functions(resources) self.assertEquals(expected, result) - convert_mock.assert_called_with('Func1', {}) + convert_mock.assert_called_with('Func1', {}, []) @patch.object(SamFunctionProvider, "_convert_lambda_function_resource") def test_must_work_for_lambda_function(self, convert_mock): @@ -219,7 +243,7 @@ def test_must_work_for_lambda_function(self, convert_mock): result = SamFunctionProvider._extract_functions(resources) self.assertEquals(expected, result) - convert_mock.assert_called_with('Func1', {"a": "b"}) + convert_mock.assert_called_with('Func1', {"a": "b"}, []) def test_must_skip_unknown_resource(self): resources = { @@ -247,7 +271,8 @@ def test_must_convert(self): "Timeout": "mytimeout", "Handler": "myhandler", "Environment": "myenvironment", - "Role": "myrole" + "Role": "myrole", + "Layers": ["Layer1", "Layer2"] } expected = Function( @@ -258,10 +283,11 @@ def test_must_convert(self): handler="myhandler", codeuri="/usr/local", environment="myenvironment", - rolearn="myrole" + rolearn="myrole", + layers=["Layer1", "Layer2"] ) - result = SamFunctionProvider._convert_sam_function_resource(name, properties) + result = SamFunctionProvider._convert_sam_function_resource(name, properties, ["Layer1", "Layer2"]) self.assertEquals(expected, result) @@ -280,10 +306,11 @@ def test_must_skip_non_existent_properties(self): handler=None, codeuri="/usr/local", environment=None, - rolearn=None + rolearn=None, + layers=[] ) - result = SamFunctionProvider._convert_sam_function_resource(name, properties) + result = SamFunctionProvider._convert_sam_function_resource(name, properties, []) self.assertEquals(expected, result) @@ -294,7 +321,7 @@ def test_must_default_missing_code_uri(self): "Runtime": "myruntime" } - result = SamFunctionProvider._convert_sam_function_resource(name, properties) + result = SamFunctionProvider._convert_sam_function_resource(name, properties, []) self.assertEquals(result.codeuri, ".") # Default value def test_must_handle_code_dict(self): @@ -307,7 +334,7 @@ def test_must_handle_code_dict(self): } } - result = SamFunctionProvider._convert_sam_function_resource(name, properties) + result = SamFunctionProvider._convert_sam_function_resource(name, properties, []) self.assertEquals(result.codeuri, ".") # Default value def test_must_handle_code_s3_uri(self): @@ -317,7 +344,7 @@ def test_must_handle_code_s3_uri(self): "CodeUri": "s3://bucket/key" } - result = SamFunctionProvider._convert_sam_function_resource(name, properties) + result = SamFunctionProvider._convert_sam_function_resource(name, properties, []) self.assertEquals(result.codeuri, ".") # Default value @@ -335,7 +362,8 @@ def test_must_convert(self): "Timeout": "mytimeout", "Handler": "myhandler", "Environment": "myenvironment", - "Role": "myrole" + "Role": "myrole", + "Layers": ["Layer1", "Layer2"] } expected = Function( @@ -346,10 +374,11 @@ def test_must_convert(self): handler="myhandler", codeuri=".", environment="myenvironment", - rolearn="myrole" + rolearn="myrole", + layers=["Layer1", "Layer2"] ) - result = SamFunctionProvider._convert_lambda_function_resource(name, properties) + result = SamFunctionProvider._convert_lambda_function_resource(name, properties, ["Layer1", "Layer2"]) self.assertEquals(expected, result) @@ -370,14 +399,76 @@ def test_must_skip_non_existent_properties(self): handler=None, codeuri=".", environment=None, - rolearn=None + rolearn=None, + layers=[] ) - result = SamFunctionProvider._convert_lambda_function_resource(name, properties) + result = SamFunctionProvider._convert_lambda_function_resource(name, properties, []) self.assertEquals(expected, result) +class TestSamFunctionProvider_parse_layer_info(TestCase): + + @parameterized.expand([ + ({ + "Function": { + "Type": "AWS::Serverless::Function", + "Properties": { + } + } + }, {"Ref": "Function"}), + ({}, {"Ref": "LayerDoesNotExist"}) + ]) + def test_raise_on_invalid_layer_resource(self, resources, layer_reference): + with self.assertRaises(InvalidLayerReference): + SamFunctionProvider._parse_layer_info([layer_reference], resources) + + def test_layers_created_from_template_resources(self): + resources = { + "Layer": { + "Type": "AWS::Lambda::LayerVersion", + "Properties": { + "Content": { + "Bucket": "bucket" + } + } + }, + "ServerlessLayer": { + "Type": "AWS::Serverless::LayerVersion", + "Properties": { + "ContentUri": "/somepath" + } + } + } + + list_of_layers = [{"Ref": "Layer"}, + {"Ref": "ServerlessLayer"}, + "arn:aws:lambda:region:account-id:layer:layer-name:1", + {"NonRef": "Something"}] + actual = SamFunctionProvider._parse_layer_info(list_of_layers, resources) + + for (actual_layer, expected_layer) in zip(actual, [LayerVersion("Layer", "."), + LayerVersion("ServerlessLayer", "/somepath"), + LayerVersion( + "arn:aws:lambda:region:account-id:layer:layer-name:1", + None)]): + self.assertEquals(actual_layer, expected_layer) + + def test_return_empty_list_on_no_layers(self): + resources = { + "Function": { + "Type": "AWS::Serverless::Function", + "Properties": { + } + } + } + + actual = SamFunctionProvider._parse_layer_info([], resources) + + self.assertEquals(actual, []) + + class TestSamFunctionProvider_get(TestCase): def test_raise_on_invalid_name(self): diff --git a/tests/unit/commands/local/start_api/test_cli.py b/tests/unit/commands/local/start_api/test_cli.py index 5f49aa05b1..13e2e02560 100644 --- a/tests/unit/commands/local/start_api/test_cli.py +++ b/tests/unit/commands/local/start_api/test_cli.py @@ -5,11 +5,14 @@ from unittest import TestCase from mock import patch, Mock +from parameterized import parameterized + from samcli.commands.local.start_api.cli import do_cli as start_api_cli -from samcli.commands.local.lib.exceptions import NoApisDefined +from samcli.commands.local.lib.exceptions import NoApisDefined, InvalidLayerReference from samcli.commands.exceptions import UserException from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.local.lib.exceptions import OverridesNotWellDefinedError +from samcli.local.docker.lambda_container import DebuggingNotSupported class TestCli(TestCase): @@ -24,9 +27,13 @@ def setUp(self): self.docker_network = "network" self.log_file = "logfile" self.skip_pull_image = True - self.profile = "profile" - self.region = "region" self.parameter_overrides = {} + self.layer_cache_basedir = "/some/layers/path" + self.force_image_build = True + self.region_name = "region" + + self.ctx_mock = Mock() + self.ctx_mock.region = self.region_name self.host = "host" self.port = 123 @@ -52,12 +59,13 @@ def test_cli_must_setup_context_and_start_service(self, local_api_service_mock, docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - aws_profile=self.profile, debug_port=self.debug_port, debug_args=self.debug_args, debugger_path=self.debugger_path, - aws_region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build, + aws_region=self.region_name) local_api_service_mock.assert_called_with(lambda_invoke_context=context_mock, port=self.port, @@ -85,16 +93,24 @@ def test_must_raise_if_no_api_defined(self, local_api_service_mock, invoke_conte expected = "Template does not have any APIs connected to Lambda functions" self.assertEquals(msg, expected) + @parameterized.expand([(InvalidSamDocumentException("bad template"), "bad template"), + (InvalidLayerReference(), "Layer References need to be of type " + "'AWS::Serverless::LayerVersion' or 'AWS::Lambda::LayerVersion'"), + (DebuggingNotSupported("Debugging not supported"), "Debugging not supported") + ]) @patch("samcli.commands.local.start_api.cli.InvokeContext") - def test_must_raise_user_exception_on_invalid_sam_template(self, invoke_context_mock): + def test_must_raise_user_exception_on_invalid_sam_template(self, + exeception_to_raise, + execption_message, + invoke_context_mock): - invoke_context_mock.side_effect = InvalidSamDocumentException("bad template") + invoke_context_mock.side_effect = exeception_to_raise with self.assertRaises(UserException) as context: self.call_cli() msg = str(context.exception) - expected = "bad template" + expected = execption_message self.assertEquals(msg, expected) @patch("samcli.commands.local.start_api.cli.InvokeContext") @@ -109,7 +125,7 @@ def test_must_raise_user_exception_on_invalid_env_vars(self, invoke_context_mock self.assertEquals(msg, expected) def call_cli(self): - start_api_cli(ctx=None, + start_api_cli(ctx=self.ctx_mock, host=self.host, port=self.port, static_dir=self.static_dir, @@ -122,6 +138,6 @@ def call_cli(self): docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) diff --git a/tests/unit/commands/local/start_lambda/test_cli.py b/tests/unit/commands/local/start_lambda/test_cli.py index 07f6a33d95..cc22295abb 100644 --- a/tests/unit/commands/local/start_lambda/test_cli.py +++ b/tests/unit/commands/local/start_lambda/test_cli.py @@ -1,10 +1,14 @@ from unittest import TestCase from mock import patch, Mock +from parameterized import parameterized + from samcli.commands.local.start_lambda.cli import do_cli as start_lambda_cli +from samcli.commands.local.lib.exceptions import InvalidLayerReference from samcli.commands.local.cli_common.user_exceptions import UserException from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.local.lib.exceptions import OverridesNotWellDefinedError +from samcli.local.docker.lambda_container import DebuggingNotSupported class TestCli(TestCase): @@ -19,9 +23,13 @@ def setUp(self): self.docker_network = "network" self.log_file = "logfile" self.skip_pull_image = True - self.profile = "profile" - self.region = "region" self.parameter_overrides = {} + self.layer_cache_basedir = "/some/layers/path" + self.force_image_build = True + self.region_name = "region" + + self.ctx_mock = Mock() + self.ctx_mock.region = self.region_name self.host = "host" self.port = 123 @@ -46,12 +54,13 @@ def test_cli_must_setup_context_and_start_service(self, local_lambda_service_moc docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - aws_profile=self.profile, debug_port=self.debug_port, debug_args=self.debug_args, debugger_path=self.debugger_path, - aws_region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build, + aws_region=self.region_name) local_lambda_service_mock.assert_called_with(lambda_invoke_context=context_mock, port=self.port, @@ -59,15 +68,24 @@ def test_cli_must_setup_context_and_start_service(self, local_lambda_service_moc service_mock.start.assert_called_with() + @parameterized.expand([(InvalidSamDocumentException("bad template"), "bad template"), + (InvalidLayerReference(), "Layer References need to be of type " + "'AWS::Serverless::LayerVersion' or 'AWS::Lambda::LayerVersion'"), + (DebuggingNotSupported("Debugging not supported"), "Debugging not supported") + ]) @patch("samcli.commands.local.start_lambda.cli.InvokeContext") - def test_must_raise_user_exception_on_invalid_sam_template(self, invoke_context_mock): - invoke_context_mock.side_effect = InvalidSamDocumentException("bad template") + def test_must_raise_user_exception_on_invalid_sam_template(self, + exeception_to_raise, + execption_message, + invoke_context_mock + ): + invoke_context_mock.side_effect = exeception_to_raise with self.assertRaises(UserException) as context: self.call_cli() msg = str(context.exception) - expected = "bad template" + expected = execption_message self.assertEquals(msg, expected) @patch("samcli.commands.local.start_lambda.cli.InvokeContext") @@ -82,7 +100,7 @@ def test_must_raise_user_exception_on_invalid_env_vars(self, invoke_context_mock self.assertEquals(msg, expected) def call_cli(self): - start_lambda_cli(ctx=None, + start_lambda_cli(ctx=self.ctx_mock, host=self.host, port=self.port, template=self.template, @@ -94,6 +112,6 @@ def call_cli(self): docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - profile=self.profile, - region=self.region, - parameter_overrides=self.parameter_overrides) + parameter_overrides=self.parameter_overrides, + layer_cache_basedir=self.layer_cache_basedir, + force_image_build=self.force_image_build) diff --git a/tests/unit/commands/validate/lib/test_sam_template_validator.py b/tests/unit/commands/validate/lib/test_sam_template_validator.py index b1086b2f37..66bd659309 100644 --- a/tests/unit/commands/validate/lib/test_sam_template_validator.py +++ b/tests/unit/commands/validate/lib/test_sam_template_validator.py @@ -119,6 +119,12 @@ def test_replace_local_codeuri(self): "Runtime": "nodejs6.10", "Timeout": 60 } + }, + "ServerlessLayerVersion": { + "Type": "AWS::Serverless::LayerVersion", + "Properties": { + "ContentUri": "./" + } } } } @@ -130,10 +136,12 @@ def test_replace_local_codeuri(self): validator._replace_local_codeuri() # check template - tempalte_resources = validator.sam_template.get("Resources") - self.assertEquals(tempalte_resources.get("ServerlessApi").get("Properties").get("DefinitionUri"), + template_resources = validator.sam_template.get("Resources") + self.assertEquals(template_resources.get("ServerlessApi").get("Properties").get("DefinitionUri"), "s3://bucket/value") - self.assertEquals(tempalte_resources.get("ServerlessFunction").get("Properties").get("CodeUri"), + self.assertEquals(template_resources.get("ServerlessFunction").get("Properties").get("CodeUri"), + "s3://bucket/value") + self.assertEquals(template_resources.get("ServerlessLayerVersion").get("Properties").get("ContentUri"), "s3://bucket/value") def test_replace_local_codeuri_when_no_codeuri_given(self): diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index 207c151c61..6be2e2b7e5 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -97,7 +97,7 @@ def setUp(self): } def test_must_write_relative_build_artifacts_path(self): - target_template_path = "/path/to/tempate.txt" + original_template_path = "/path/to/tempate.txt" built_artifacts = { "MyFunction1": "/path/to/build/MyFunction1", "MyFunction2": "/path/to/build/MyFunction2" @@ -126,7 +126,7 @@ def test_must_write_relative_build_artifacts_path(self): } } - actual = self.builder.update_template(self.template_dict, target_template_path, built_artifacts) + actual = self.builder.update_template(self.template_dict, original_template_path, built_artifacts) self.assertEquals(actual, expected_result) def test_must_skip_if_no_artifacts(self): diff --git a/tests/unit/lib/utils/test_codeuri.py b/tests/unit/lib/utils/test_codeuri.py new file mode 100644 index 0000000000..44dc8e9449 --- /dev/null +++ b/tests/unit/lib/utils/test_codeuri.py @@ -0,0 +1,68 @@ +import os +from unittest import TestCase +from parameterized import parameterized + +from samcli.lib.utils.codeuri import resolve_code_path + + +class TestLocalLambda_get_code_path(TestCase): + + def setUp(self): + self.cwd = "/my/current/working/directory" + self.relative_codeuri = "./my/path" + self.absolute_codeuri = "/home/foo/bar" # Some absolute path to use + self.os_cwd = os.getcwd() + + @parameterized.expand([ + ("."), + ("") + ]) + def test_must_resolve_present_cwd(self, cwd_path): + codeuri = self.relative_codeuri + expected = os.path.normpath(os.path.join(self.os_cwd, codeuri)) + + actual = resolve_code_path(cwd_path, codeuri) + self.assertEquals(expected, actual) + self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") + + @parameterized.expand([ + ("var/task"), + ("some/dir") + ]) + def test_must_resolve_relative_cwd(self, cwd_path): + + codeuri = self.relative_codeuri + + abs_cwd = os.path.abspath(cwd_path) + expected = os.path.normpath(os.path.join(abs_cwd, codeuri)) + + actual = resolve_code_path(cwd_path, codeuri) + self.assertEquals(expected, actual) + self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") + + @parameterized.expand([ + (""), + ("."), + ("hello"), + ("a/b/c/d"), + ("../../c/d/e") + ]) + def test_must_resolve_relative_codeuri(self, codeuri): + + expected = os.path.normpath(os.path.join(self.cwd, codeuri)) + + actual = resolve_code_path(self.cwd, codeuri) + self.assertEquals(expected, actual) + self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") + + @parameterized.expand([ + ("/a/b/c"), + ("/") + ]) + def test_must_resolve_absolute_codeuri(self, codeuri): + + expected = codeuri # CodeUri must be return as is for absolute paths + + actual = resolve_code_path(None, codeuri) + self.assertEquals(expected, actual) + self.assertTrue(os.path.isabs(actual), "Result must be an absolute path") diff --git a/tests/unit/lib/utils/test_progressbar.py b/tests/unit/lib/utils/test_progressbar.py new file mode 100644 index 0000000000..00a30073a5 --- /dev/null +++ b/tests/unit/lib/utils/test_progressbar.py @@ -0,0 +1,18 @@ +from unittest import TestCase +from mock import patch, Mock + +from samcli.lib.utils.progressbar import progressbar + + +class TestProgressBar(TestCase): + + @patch('samcli.lib.utils.progressbar.click') + def test_creating_progressbar(self, click_patch): + progressbar_mock = Mock() + click_patch.progressbar.return_value = progressbar_mock + + actual = progressbar(100, 'this is a label') + + self.assertEquals(actual, progressbar_mock) + + click_patch.progressbar.assert_called_with(length=100, label='this is a label', show_pos=True) diff --git a/tests/unit/lib/utils/test_tar.py b/tests/unit/lib/utils/test_tar.py new file mode 100644 index 0000000000..ffc613af56 --- /dev/null +++ b/tests/unit/lib/utils/test_tar.py @@ -0,0 +1,28 @@ +from unittest import TestCase +from mock import Mock, patch, call + +from samcli.lib.utils.tar import create_tarball + + +class TestTar(TestCase): + + @patch("samcli.lib.utils.tar.tarfile.open") + @patch("samcli.lib.utils.tar.TemporaryFile") + def test_generating_tarball(self, temporary_file_patch, tarfile_open_patch): + temp_file_mock = Mock() + temporary_file_patch.return_value = temp_file_mock + + tarfile_file_mock = Mock() + tarfile_open_patch.return_value.__enter__.return_value = tarfile_file_mock + + with create_tarball({"/some/path": "/layer1", "/some/dockerfile/path": "/Dockerfile"}) as acutal: + self.assertEquals(acutal, temp_file_mock) + + tarfile_file_mock.add.assert_called() + tarfile_file_mock.add.assert_has_calls([call("/some/path", arcname="/layer1"), + call("/some/dockerfile/path", arcname="/Dockerfile")], any_order=True) + + temp_file_mock.flush.assert_called_once() + temp_file_mock.seek.assert_called_once_with(0) + temp_file_mock.close.assert_called_once() + tarfile_open_patch.assert_called_once_with(fileobj=temp_file_mock, mode='w:gz') diff --git a/tests/unit/local/docker/test_lambda_container.py b/tests/unit/local/docker/test_lambda_container.py index 0e2b6b8ef5..58675affef 100644 --- a/tests/unit/local/docker/test_lambda_container.py +++ b/tests/unit/local/docker/test_lambda_container.py @@ -3,11 +3,11 @@ """ from unittest import TestCase -from mock import patch +from mock import patch, Mock from parameterized import parameterized, param from samcli.commands.local.lib.debug_context import DebugContext -from samcli.local.docker.lambda_container import LambdaContainer, Runtime +from samcli.local.docker.lambda_container import LambdaContainer, Runtime, DebuggingNotSupported RUNTIMES_WITH_ENTRYPOINT = [Runtime.java8.value, Runtime.go1x.value, @@ -56,9 +56,13 @@ def test_must_configure_container_properly(self, get_additional_options_mock.return_value = addtl_options get_additional_volumes_mock.return_value = addtl_volumes + image_builder_mock = Mock() + container = LambdaContainer(self.runtime, self.handler, self.code_dir, + layers=[], + image_builder=image_builder_mock, env_vars=self.env_var, memory_mb=self.memory_mb, debug_options=self.debug_options) @@ -72,7 +76,7 @@ def test_must_configure_container_properly(self, self.assertEquals(self.env_var, container._env_vars) self.assertEquals(self.memory_mb, container._memory_limit_mb) - get_image_mock.assert_called_with(self.runtime) + get_image_mock.assert_called_with(image_builder_mock, self.runtime, []) get_exposed_ports_mock.assert_called_with(self.debug_options) get_entry_point_mock.assert_called_with(self.runtime, self.debug_options) get_additional_options_mock.assert_called_with(self.runtime, self.debug_options) @@ -82,8 +86,10 @@ def test_must_fail_for_unsupported_runtime(self): runtime = "foo" + image_builder_mock = Mock() + with self.assertRaises(ValueError) as context: - LambdaContainer(runtime, self.handler, self.code_dir) + LambdaContainer(runtime, self.handler, self.code_dir, [], image_builder_mock) self.assertEquals(str(context.exception), "Unsupported Lambda runtime foo") @@ -107,10 +113,12 @@ class TestLambdaContainer_get_image(TestCase): def test_must_return_lambci_image(self): - runtime = "foo" expected = "lambci/lambda:foo" - self.assertEquals(LambdaContainer._get_image(runtime), expected) + image_builder = Mock() + image_builder.build.return_value = expected + + self.assertEquals(LambdaContainer._get_image(image_builder, 'foo', []), expected) class TestLambdaContainer_get_entry_point(TestCase): @@ -128,12 +136,12 @@ def test_must_skip_if_debug_port_is_not_specified(self): @parameterized.expand([param(r) for r in ALL_RUNTIMES]) def test_must_provide_entrypoint_for_certain_runtimes_only(self, runtime): - result = LambdaContainer._get_entry_point(runtime, self.debug_options) - if runtime in RUNTIMES_WITH_ENTRYPOINT: + result = LambdaContainer._get_entry_point(runtime, self.debug_options) self.assertIsNotNone(result, "{} runtime must provide entrypoint".format(runtime)) else: - self.assertIsNone(result, "{} runtime must NOT provide entrypoint".format(runtime)) + with self.assertRaises(DebuggingNotSupported): + LambdaContainer._get_entry_point(runtime, self.debug_options) @parameterized.expand([param(r) for r in RUNTIMES_WITH_ENTRYPOINT]) def test_debug_arg_must_be_split_by_spaces_and_appended_to_entrypoint(self, runtime): diff --git a/tests/unit/local/docker/test_lambda_image.py b/tests/unit/local/docker/test_lambda_image.py new file mode 100644 index 0000000000..7a50aef1ee --- /dev/null +++ b/tests/unit/local/docker/test_lambda_image.py @@ -0,0 +1,269 @@ +from unittest import TestCase +from mock import patch, Mock, mock_open + +from docker.errors import ImageNotFound, BuildError, APIError + +from samcli.local.docker.lambda_image import LambdaImage +from samcli.commands.local.cli_common.user_exceptions import ImageBuildException + + +class TestLambdaImage(TestCase): + + def test_initialization_without_defaults(self): + lambda_image = LambdaImage("layer_downloader", False, False, docker_client="docker_client") + + self.assertEquals(lambda_image.layer_downloader, "layer_downloader") + self.assertFalse(lambda_image.skip_pull_image) + self.assertFalse(lambda_image.force_image_build) + self.assertEquals(lambda_image.docker_client, "docker_client") + + @patch("samcli.local.docker.lambda_image.docker") + def test_initialization_with_defaults(self, docker_patch): + docker_client_mock = Mock() + docker_patch.from_env.return_value = docker_client_mock + + lambda_image = LambdaImage("layer_downloader", False, False) + + self.assertEquals(lambda_image.layer_downloader, "layer_downloader") + self.assertFalse(lambda_image.skip_pull_image) + self.assertFalse(lambda_image.force_image_build) + self.assertEquals(lambda_image.docker_client, docker_client_mock) + + def test_building_image_with_no_layers(self): + docker_client_mock = Mock() + + lambda_image = LambdaImage("layer_downloader", False, False, docker_client=docker_client_mock) + + self.assertEquals(lambda_image.build("python3.6", []), "lambci/lambda:python3.6") + + @patch("samcli.local.docker.lambda_image.LambdaImage._build_image") + @patch("samcli.local.docker.lambda_image.LambdaImage._generate_docker_image_version") + def test_not_building_image_that_already_exists(self, + generate_docker_image_version_patch, + build_image_patch): + layer_downloader_mock = Mock() + layer_mock = Mock() + layer_mock.name = "layers1" + layer_mock.is_defined_within_template = False + layer_downloader_mock.download_all.return_value = [layer_mock] + + generate_docker_image_version_patch.return_value = "image-version" + + docker_client_mock = Mock() + docker_client_mock.images.get.return_value = Mock() + + lambda_image = LambdaImage(layer_downloader_mock, False, False, docker_client=docker_client_mock) + actual_image_id = lambda_image.build("python3.6", [layer_mock]) + + self.assertEquals(actual_image_id, "samcli/lambda:image-version") + + layer_downloader_mock.download_all.assert_called_once_with([layer_mock], False) + generate_docker_image_version_patch.assert_called_once_with([layer_mock], "python3.6") + docker_client_mock.images.get.assert_called_once_with("samcli/lambda:image-version") + build_image_patch.assert_not_called() + + @patch("samcli.local.docker.lambda_image.LambdaImage._build_image") + @patch("samcli.local.docker.lambda_image.LambdaImage._generate_docker_image_version") + def test_force_building_image_that_doesnt_already_exists(self, + generate_docker_image_version_patch, + build_image_patch): + layer_downloader_mock = Mock() + layer_downloader_mock.download_all.return_value = ["layers1"] + + generate_docker_image_version_patch.return_value = "image-version" + + docker_client_mock = Mock() + docker_client_mock.images.get.side_effect = ImageNotFound("image not found") + + lambda_image = LambdaImage(layer_downloader_mock, False, True, docker_client=docker_client_mock) + actual_image_id = lambda_image.build("python3.6", ["layers1"]) + + self.assertEquals(actual_image_id, "samcli/lambda:image-version") + + layer_downloader_mock.download_all.assert_called_once_with(["layers1"], True) + generate_docker_image_version_patch.assert_called_once_with(["layers1"], "python3.6") + docker_client_mock.images.get.assert_called_once_with("samcli/lambda:image-version") + build_image_patch.assert_called_once_with("lambci/lambda:python3.6", "samcli/lambda:image-version", ["layers1"]) + + @patch("samcli.local.docker.lambda_image.LambdaImage._build_image") + @patch("samcli.local.docker.lambda_image.LambdaImage._generate_docker_image_version") + def test_not_force_building_image_that_doesnt_already_exists(self, + generate_docker_image_version_patch, + build_image_patch): + layer_downloader_mock = Mock() + layer_downloader_mock.download_all.return_value = ["layers1"] + + generate_docker_image_version_patch.return_value = "image-version" + + docker_client_mock = Mock() + docker_client_mock.images.get.side_effect = ImageNotFound("image not found") + + lambda_image = LambdaImage(layer_downloader_mock, False, False, docker_client=docker_client_mock) + actual_image_id = lambda_image.build("python3.6", ["layers1"]) + + self.assertEquals(actual_image_id, "samcli/lambda:image-version") + + layer_downloader_mock.download_all.assert_called_once_with(["layers1"], False) + generate_docker_image_version_patch.assert_called_once_with(["layers1"], "python3.6") + docker_client_mock.images.get.assert_called_once_with("samcli/lambda:image-version") + build_image_patch.assert_called_once_with("lambci/lambda:python3.6", "samcli/lambda:image-version", ["layers1"]) + + @patch("samcli.local.docker.lambda_image.hashlib") + def test_generate_docker_image_version(self, hashlib_patch): + haslib_sha256_mock = Mock() + hashlib_patch.sha256.return_value = haslib_sha256_mock + haslib_sha256_mock.hexdigest.return_value = "thisisahexdigestofshahash" + + layer_mock = Mock() + layer_mock.name = 'layer1' + + image_version = LambdaImage._generate_docker_image_version([layer_mock], 'runtime') + + self.assertEquals(image_version, "runtime-thisisahexdigestofshahash") + + hashlib_patch.sha256.assert_called_once_with(b'layer1') + + @patch("samcli.local.docker.lambda_image.docker") + def test_generate_dockerfile(self, docker_patch): + docker_client_mock = Mock() + docker_patch.from_env.return_value = docker_client_mock + + expected_docker_file = "FROM python\nADD --chown=sbx_user1051:495 layer1 /opt\n" + + layer_mock = Mock() + layer_mock.name = "layer1" + + self.assertEquals(LambdaImage._generate_dockerfile("python", [layer_mock]), expected_docker_file) + + @patch("samcli.local.docker.lambda_image.create_tarball") + @patch("samcli.local.docker.lambda_image.uuid") + @patch("samcli.local.docker.lambda_image.Path") + @patch("samcli.local.docker.lambda_image.LambdaImage._generate_dockerfile") + def test_build_image(self, generate_dockerfile_patch, path_patch, uuid_patch, create_tarball_patch): + uuid_patch.uuid4.return_value = "uuid" + generate_dockerfile_patch.return_value = "Dockerfile content" + + docker_full_path_mock = Mock() + docker_full_path_mock.exists.return_value = True + path_patch.return_value = docker_full_path_mock + + docker_client_mock = Mock() + layer_downloader_mock = Mock() + layer_downloader_mock.layer_cache = "cached layers" + + tarball_fileobj = Mock() + create_tarball_patch.return_value.__enter__.return_value = tarball_fileobj + + layer_version1 = Mock() + layer_version1.codeuri = "somevalue" + layer_version1.name = "name" + + dockerfile_mock = Mock() + m = mock_open(dockerfile_mock) + with patch("samcli.local.docker.lambda_image.open", m): + LambdaImage(layer_downloader_mock, True, False, docker_client=docker_client_mock)\ + ._build_image("base_image", "docker_tag", [layer_version1]) + + handle = m() + handle.write.assert_called_with("Dockerfile content") + path_patch.assert_called_once_with("cached layers", "dockerfile_uuid") + docker_client_mock.images.build.assert_called_once_with(fileobj=tarball_fileobj, + rm=True, + tag="docker_tag", + pull=False, + custom_context=True, + encoding='gzip') + + docker_full_path_mock.unlink.assert_called_once() + + @patch("samcli.local.docker.lambda_image.create_tarball") + @patch("samcli.local.docker.lambda_image.uuid") + @patch("samcli.local.docker.lambda_image.Path") + @patch("samcli.local.docker.lambda_image.LambdaImage._generate_dockerfile") + def test_build_image_fails_with_BuildError(self, + generate_dockerfile_patch, + path_patch, + uuid_patch, + create_tarball_patch): + uuid_patch.uuid4.return_value = "uuid" + generate_dockerfile_patch.return_value = "Dockerfile content" + + docker_full_path_mock = Mock() + docker_full_path_mock.exists.return_value = False + path_patch.return_value = docker_full_path_mock + + docker_client_mock = Mock() + docker_client_mock.images.build.side_effect = BuildError("buildError", "buildlog") + layer_downloader_mock = Mock() + layer_downloader_mock.layer_cache = "cached layers" + + tarball_fileobj = Mock() + create_tarball_patch.return_value.__enter__.return_value = tarball_fileobj + + layer_version1 = Mock() + layer_version1.codeuri = "somevalue" + layer_version1.name = "name" + + dockerfile_mock = Mock() + m = mock_open(dockerfile_mock) + with patch("samcli.local.docker.lambda_image.open", m): + with self.assertRaises(ImageBuildException): + LambdaImage(layer_downloader_mock, True, False, docker_client=docker_client_mock) \ + ._build_image("base_image", "docker_tag", [layer_version1]) + + handle = m() + handle.write.assert_called_with("Dockerfile content") + path_patch.assert_called_once_with("cached layers", "dockerfile_uuid") + docker_client_mock.images.build.assert_called_once_with(fileobj=tarball_fileobj, + rm=True, + tag="docker_tag", + pull=False, + custom_context=True, + encoding='gzip') + + docker_full_path_mock.unlink.assert_not_called() + + @patch("samcli.local.docker.lambda_image.create_tarball") + @patch("samcli.local.docker.lambda_image.uuid") + @patch("samcli.local.docker.lambda_image.Path") + @patch("samcli.local.docker.lambda_image.LambdaImage._generate_dockerfile") + def test_build_image_fails_with_ApiError(self, + generate_dockerfile_patch, + path_patch, + uuid_patch, + create_tarball_patch): + uuid_patch.uuid4.return_value = "uuid" + generate_dockerfile_patch.return_value = "Dockerfile content" + + docker_full_path_mock = Mock() + path_patch.return_value = docker_full_path_mock + + docker_client_mock = Mock() + docker_client_mock.images.build.side_effect = APIError("apiError") + layer_downloader_mock = Mock() + layer_downloader_mock.layer_cache = "cached layers" + + tarball_fileobj = Mock() + create_tarball_patch.return_value.__enter__.return_value = tarball_fileobj + + layer_version1 = Mock() + layer_version1.codeuri = "somevalue" + layer_version1.name = "name" + + dockerfile_mock = Mock() + m = mock_open(dockerfile_mock) + with patch("samcli.local.docker.lambda_image.open", m): + with self.assertRaises(ImageBuildException): + LambdaImage(layer_downloader_mock, True, False, docker_client=docker_client_mock) \ + ._build_image("base_image", "docker_tag", [layer_version1]) + + handle = m() + handle.write.assert_called_with("Dockerfile content") + path_patch.assert_called_once_with("cached layers", "dockerfile_uuid") + docker_client_mock.images.build.assert_called_once_with(fileobj=tarball_fileobj, + rm=True, + tag="docker_tag", + pull=False, + custom_context=True, + encoding='gzip') + docker_full_path_mock.unlink.assert_called_once() diff --git a/tests/unit/local/docker/test_manager.py b/tests/unit/local/docker/test_manager.py index 4002d6635f..4169b99bea 100644 --- a/tests/unit/local/docker/test_manager.py +++ b/tests/unit/local/docker/test_manager.py @@ -68,6 +68,25 @@ def test_must_pull_image_if_image_exist_and_no_skip(self): self.manager.pull_image.assert_called_with(self.image_name) self.container_mock.start.assert_called_with(input_data=input_data) + def test_must_not_pull_image_if_image_is_samcli_lambda_image(self): + input_data = "input data" + + self.manager.has_image = Mock() + self.manager.pull_image = Mock() + + # Assume the image exist. + self.manager.has_image.return_value = True + # And, don't skip pulling => Pull again + self.manager.skip_pull_image = False + + self.container_mock.image = "samcli/lambda" + + self.manager.run(self.container_mock, input_data) + + self.manager.has_image.assert_called_with("samcli/lambda") + self.manager.pull_image.assert_not_called() + self.container_mock.start.assert_called_with(input_data=input_data) + def test_must_not_pull_image_if_asked_to_skip(self): input_data = "input data" diff --git a/tests/unit/local/lambdafn/test_config.py b/tests/unit/local/lambdafn/test_config.py index f52b5d45eb..4f62ffa81a 100644 --- a/tests/unit/local/lambdafn/test_config.py +++ b/tests/unit/local/lambdafn/test_config.py @@ -18,15 +18,17 @@ def setUp(self): self.memory = 1234 self.timeout = 34 self.env_vars_mock = Mock() + self.layers = ['layer1'] def test_init_with_env_vars(self): - config = FunctionConfig(self.name, self.runtime, self.handler, self.code_path, + config = FunctionConfig(self.name, self.runtime, self.handler, self.code_path, self.layers, memory=self.memory, timeout=self.timeout, env_vars=self.env_vars_mock) self.assertEquals(config.name, self.name) self.assertEquals(config.runtime, self.runtime) self.assertEquals(config.handler, self.handler) self.assertEquals(config.code_abs_path, self.code_path) + self.assertEquals(config.layers, self.layers) self.assertEquals(config.memory, self.memory) self.assertEquals(config.timeout, self.timeout) self.assertEquals(config.env_vars, self.env_vars_mock) @@ -36,12 +38,13 @@ def test_init_with_env_vars(self): self.assertEquals(self.env_vars_mock.timeout, self.timeout) def test_init_without_optional_values(self): - config = FunctionConfig(self.name, self.runtime, self.handler, self.code_path) + config = FunctionConfig(self.name, self.runtime, self.handler, self.code_path, self.layers) self.assertEquals(config.name, self.name) self.assertEquals(config.runtime, self.runtime) self.assertEquals(config.handler, self.handler) self.assertEquals(config.code_abs_path, self.code_path) + self.assertEquals(config.layers, self.layers) self.assertEquals(config.memory, self.DEFAULT_MEMORY) self.assertEquals(config.timeout, self.DEFAULT_TIMEOUT) self.assertIsNotNone(config.env_vars) diff --git a/tests/unit/local/lambdafn/test_runtime.py b/tests/unit/local/lambdafn/test_runtime.py index 6a6ef35a6d..f3079ecb10 100644 --- a/tests/unit/local/lambdafn/test_runtime.py +++ b/tests/unit/local/lambdafn/test_runtime.py @@ -23,7 +23,8 @@ def setUp(self): self.lang = "runtime" self.handler = "handler" self.code_path = "code-path" - self.func_config = FunctionConfig(self.name, self.lang, self.handler, self.code_path) + self.layers = [] + self.func_config = FunctionConfig(self.name, self.lang, self.handler, self.code_path, self.layers) self.env_vars = Mock() self.func_config.env_vars = self.env_vars @@ -39,8 +40,9 @@ def test_must_run_container_and_wait_for_logs(self, LambdaContainerMock): container = Mock() timer = Mock() debug_options = Mock() + lambda_image_mock = Mock() - self.runtime = LambdaRuntime(self.manager_mock) + self.runtime = LambdaRuntime(self.manager_mock, lambda_image_mock) # Using MagicMock to mock the context manager self.runtime._get_code_dir = MagicMock() @@ -68,7 +70,7 @@ def test_must_run_container_and_wait_for_logs(self, LambdaContainerMock): self.runtime._get_code_dir.assert_called_with(self.code_path) # Make sure the container is created with proper values - LambdaContainerMock.assert_called_with(self.lang, self.handler, code_dir, + LambdaContainerMock.assert_called_with(self.lang, self.handler, code_dir, self.layers, lambda_image_mock, memory_mb=self.DEFAULT_MEMORY, env_vars=self.env_var_value, debug_options=debug_options) @@ -89,8 +91,9 @@ def test_exception_from_run_must_trigger_cleanup(self, LambdaContainerMock): stderr = "stderr" container = Mock() timer = Mock() + layer_downloader = Mock() - self.runtime = LambdaRuntime(self.manager_mock) + self.runtime = LambdaRuntime(self.manager_mock, layer_downloader) # Using MagicMock to mock the context manager self.runtime._get_code_dir = MagicMock() @@ -129,8 +132,9 @@ def test_exception_from_wait_for_logs_must_trigger_cleanup(self, LambdaContainer container = Mock() timer = Mock() debug_options = Mock() + layer_downloader = Mock() - self.runtime = LambdaRuntime(self.manager_mock) + self.runtime = LambdaRuntime(self.manager_mock, layer_downloader) # Using MagicMock to mock the context manager self.runtime._get_code_dir = MagicMock() @@ -167,8 +171,9 @@ def test_keyboard_interrupt_must_not_raise(self, LambdaContainerMock): stdout = "stdout" stderr = "stderr" container = Mock() + layer_downloader = Mock() - self.runtime = LambdaRuntime(self.manager_mock) + self.runtime = LambdaRuntime(self.manager_mock, layer_downloader) # Using MagicMock to mock the context manager self.runtime._get_code_dir = MagicMock() @@ -201,7 +206,8 @@ def setUp(self): self.container = Mock() self.manager_mock = Mock() - self.runtime = LambdaRuntime(self.manager_mock) + self.layer_downloader = Mock() + self.runtime = LambdaRuntime(self.manager_mock, self.layer_downloader) @patch("samcli.local.lambdafn.runtime.threading") @patch("samcli.local.lambdafn.runtime.signal") @@ -274,7 +280,8 @@ class TestLambdaRuntime_get_code_dir(TestCase): def setUp(self): self.manager_mock = Mock() - self.runtime = LambdaRuntime(self.manager_mock) + self.layer_downloader = Mock() + self.runtime = LambdaRuntime(self.manager_mock, self.layer_downloader) @parameterized.expand([ (".zip"), diff --git a/tests/unit/local/lambdafn/test_zip.py b/tests/unit/local/lambdafn/test_zip.py index 12e0969a55..973990a500 100644 --- a/tests/unit/local/lambdafn/test_zip.py +++ b/tests/unit/local/lambdafn/test_zip.py @@ -1,16 +1,16 @@ - import stat import zipfile import os import shutil - -from samcli.local.lambdafn.zip import unzip - -from tempfile import NamedTemporaryFile, mkdtemp -from contextlib import contextmanager from unittest import TestCase +from contextlib import contextmanager +from tempfile import NamedTemporaryFile, mkdtemp +from mock import Mock, patch + from nose_parameterized import parameterized, param +from samcli.local.lambdafn.zip import unzip, unzip_from_uri, _override_permissions + class TestUnzipWithPermissions(TestCase): @@ -77,3 +77,90 @@ def _temp_dir(self): finally: if name: shutil.rmtree(name) + + +class TestUnzipFromUri(TestCase): + + @patch('samcli.local.lambdafn.zip.unzip') + @patch('samcli.local.lambdafn.zip.Path') + @patch('samcli.local.lambdafn.zip.progressbar') + @patch('samcli.local.lambdafn.zip.requests') + @patch('samcli.local.lambdafn.zip.open') + def test_successfully_unzip_from_uri(self, open_patch, requests_patch, progressbar_patch, path_patch, unzip_patch): + get_request_mock = Mock() + get_request_mock.headers = {"Content-length": "200"} + get_request_mock.iter_content.return_value = [b'data1'] + requests_patch.get.return_value = get_request_mock + + file_mock = Mock() + open_patch.return_value.__enter__.return_value = file_mock + + progressbar_mock = Mock() + progressbar_patch.return_value.__enter__.return_value = progressbar_mock + + path_mock = Mock() + path_mock.exists.return_value = True + path_patch.return_value = path_mock + + unzip_from_uri('uri', 'layer_zip_path', 'output_zip_dir', 'layer_arn') + + requests_patch.get.assert_called_with('uri', stream=True) + get_request_mock.iter_content.assert_called_with(chunk_size=None) + open_patch.assert_called_with('layer_zip_path', 'wb') + file_mock.write.assert_called_with(b'data1') + progressbar_mock.update.assert_called_with(5) + path_patch.assert_called_with('layer_zip_path') + path_mock.unlink.assert_called() + unzip_patch.assert_called_with('layer_zip_path', 'output_zip_dir', permission=0o700) + + @patch('samcli.local.lambdafn.zip.unzip') + @patch('samcli.local.lambdafn.zip.Path') + @patch('samcli.local.lambdafn.zip.progressbar') + @patch('samcli.local.lambdafn.zip.requests') + @patch('samcli.local.lambdafn.zip.open') + def test_not_unlink_file_when_file_doesnt_exist(self, + open_patch, + requests_patch, + progressbar_patch, + path_patch, + unzip_patch): + get_request_mock = Mock() + get_request_mock.headers = {"Content-length": "200"} + get_request_mock.iter_content.return_value = [b'data1'] + requests_patch.get.return_value = get_request_mock + + file_mock = Mock() + open_patch.return_value.__enter__.return_value = file_mock + + progressbar_mock = Mock() + progressbar_patch.return_value.__enter__.return_value = progressbar_mock + + path_mock = Mock() + path_mock.exists.return_value = False + path_patch.return_value = path_mock + + unzip_from_uri('uri', 'layer_zip_path', 'output_zip_dir', 'layer_arn') + + requests_patch.get.assert_called_with('uri', stream=True) + get_request_mock.iter_content.assert_called_with(chunk_size=None) + open_patch.assert_called_with('layer_zip_path', 'wb') + file_mock.write.assert_called_with(b'data1') + progressbar_mock.update.assert_called_with(5) + path_patch.assert_called_with('layer_zip_path') + path_mock.unlink.assert_not_called() + unzip_patch.assert_called_with('layer_zip_path', 'output_zip_dir', permission=0o700) + + +class TestOverridePermissions(TestCase): + + @patch('samcli.local.lambdafn.zip.os') + def test_must_override_permissions(self, os_patch): + _override_permissions(path="./home", permission=0o700) + + os_patch.chmod.assert_called_once_with("./home", 0o700) + + @patch('samcli.local.lambdafn.zip.os') + def test_must_not_override_permissions(self, os_patch): + _override_permissions(path="./home", permission=None) + + os_patch.chmod.assert_not_called() diff --git a/tests/unit/local/layers/__init__.py b/tests/unit/local/layers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/local/layers/test_download_layers.py b/tests/unit/local/layers/test_download_layers.py new file mode 100644 index 0000000000..a148c98b0b --- /dev/null +++ b/tests/unit/local/layers/test_download_layers.py @@ -0,0 +1,197 @@ +from unittest import TestCase +from mock import patch, Mock, call + +from botocore.exceptions import NoCredentialsError, ClientError + +from samcli.local.layers.layer_downloader import LayerDownloader +from samcli.commands.local.cli_common.user_exceptions import CredentialsRequired, ResourceNotFound + + +class TestDownloadLayers(TestCase): + + def test_initialization(self): + download_layers = LayerDownloader("/some/path", ".") + + self.assertEquals(download_layers.layer_cache, "/some/path") + + @patch("samcli.local.layers.layer_downloader.LayerDownloader.download") + def test_download_all_without_force(self, download_patch): + download_patch.side_effect = ['/home/layer1', '/home/layer2'] + + download_layers = LayerDownloader("/home", ".") + + acutal_results = download_layers.download_all(['layer1', 'layer2']) + + self.assertEquals(acutal_results, ['/home/layer1', '/home/layer2']) + + download_patch.assert_has_calls([call('layer1', False), call("layer2", False)]) + + @patch("samcli.local.layers.layer_downloader.LayerDownloader.download") + def test_download_all_with_force(self, download_patch): + download_patch.side_effect = ['/home/layer1', '/home/layer2'] + + download_layers = LayerDownloader("/home", ".") + + acutal_results = download_layers.download_all(['layer1', 'layer2'], force=True) + + self.assertEquals(acutal_results, ['/home/layer1', '/home/layer2']) + + download_patch.assert_has_calls([call('layer1', True), call("layer2", True)]) + + @patch("samcli.local.layers.layer_downloader.LayerDownloader._create_cache") + @patch("samcli.local.layers.layer_downloader.LayerDownloader._is_layer_cached") + def test_download_layer_that_is_cached(self, is_layer_cached_patch, create_cache_patch): + is_layer_cached_patch.return_value = True + + download_layers = LayerDownloader("/home", ".") + + layer_mock = Mock() + layer_mock.is_defined_within_template = False + layer_mock.name = "layer1" + + actual = download_layers.download(layer_mock) + + self.assertEquals(actual.codeuri, '/home/layer1') + + create_cache_patch.assert_called_once_with("/home") + + @patch("samcli.local.layers.layer_downloader.resolve_code_path") + @patch("samcli.local.layers.layer_downloader.LayerDownloader._create_cache") + def test_download_layer_that_was_template_defined(self, create_cache_patch, resolve_code_path_patch): + + download_layers = LayerDownloader("/home", ".") + + layer_mock = Mock() + layer_mock.is_template_defined = True + layer_mock.name = "layer1" + layer_mock.codeuri = "/some/custom/path" + + resolve_code_path_patch.return_value = './some/custom/path' + + actual = download_layers.download(layer_mock) + + self.assertEquals(actual.codeuri, './some/custom/path') + + create_cache_patch.assert_not_called() + resolve_code_path_patch.assert_called_once_with(".", "/some/custom/path") + + @patch("samcli.local.layers.layer_downloader.unzip_from_uri") + @patch("samcli.local.layers.layer_downloader.LayerDownloader._fetch_layer_uri") + @patch("samcli.local.layers.layer_downloader.LayerDownloader._create_cache") + @patch("samcli.local.layers.layer_downloader.LayerDownloader._is_layer_cached") + def test_download_layer(self, is_layer_cached_patch, create_cache_patch, + fetch_layer_uri_patch, unzip_from_uri_patch): + is_layer_cached_patch.return_value = False + + download_layers = LayerDownloader("/home", ".") + + layer_mock = Mock() + layer_mock.is_defined_within_template = False + layer_mock.name = "layer1" + layer_mock.arn = "arn:layer:layer1:1" + layer_mock.layer_arn = "arn:layer:layer1" + + fetch_layer_uri_patch.return_value = "layer/uri" + + actual = download_layers.download(layer_mock) + + self.assertEquals(actual.codeuri, "/home/layer1") + + create_cache_patch.assert_called_once_with("/home") + fetch_layer_uri_patch.assert_called_once_with(layer_mock) + unzip_from_uri_patch.assert_called_once_with("layer/uri", + '/home/layer1.zip', + unzip_output_dir='/home/layer1', + progressbar_label="Downloading arn:layer:layer1") + + def test_layer_is_cached(self): + download_layers = LayerDownloader("/", ".") + + layer_path = Mock() + layer_path.exists.return_value = True + + self.assertTrue(download_layers._is_layer_cached(layer_path)) + + def test_layer_is_not_cached(self): + download_layers = LayerDownloader("/", ".") + + layer_path = Mock() + layer_path.exists.return_value = False + + self.assertFalse(download_layers._is_layer_cached(layer_path)) + + @patch("samcli.local.layers.layer_downloader.Path") + def test_create_cache(self, path_patch): + cache_path_mock = Mock() + path_patch.return_value = cache_path_mock + + self.assertIsNone(LayerDownloader._create_cache("./home")) + + path_patch.assert_called_once_with("./home") + cache_path_mock.mkdir.assert_called_once_with(parents=True, exist_ok=True, mode=0o700) + + +class TestLayerDownloader_fetch_layer_uri(TestCase): + + def test_fetch_layer_uri_is_successful(self): + lambda_client_mock = Mock() + lambda_client_mock.get_layer_version.return_value = {"Content": {"Location": "some/uri"}} + download_layers = LayerDownloader("/", ".", lambda_client_mock) + + layer = Mock() + layer.layer_arn = "arn" + layer.version = 1 + actual_uri = download_layers._fetch_layer_uri(layer=layer) + + self.assertEquals(actual_uri, "some/uri") + + def test_fetch_layer_uri_fails_with_no_creds(self): + lambda_client_mock = Mock() + lambda_client_mock.get_layer_version.side_effect = NoCredentialsError() + download_layers = LayerDownloader("/", ".", lambda_client_mock) + + layer = Mock() + layer.layer_arn = "arn" + layer.version = 1 + + with self.assertRaises(CredentialsRequired): + download_layers._fetch_layer_uri(layer=layer) + + def test_fetch_layer_uri_fails_with_AccessDeniedException(self): + lambda_client_mock = Mock() + lambda_client_mock.get_layer_version.side_effect = ClientError( + error_response={'Error': {'Code': 'AccessDeniedException'}}, operation_name="lambda") + download_layers = LayerDownloader("/", ".", lambda_client_mock) + + layer = Mock() + layer.layer_arn = "arn" + layer.version = 1 + + with self.assertRaises(CredentialsRequired): + download_layers._fetch_layer_uri(layer=layer) + + def test_fetch_layer_uri_fails_with_ResourceNotFoundException(self): + lambda_client_mock = Mock() + lambda_client_mock.get_layer_version.side_effect = ClientError( + error_response={'Error': {'Code': 'ResourceNotFoundException'}}, operation_name="lambda") + download_layers = LayerDownloader("/", ".", lambda_client_mock) + + layer = Mock() + layer.layer_arn = "arn" + layer.version = 1 + + with self.assertRaises(ResourceNotFound): + download_layers._fetch_layer_uri(layer=layer) + + def test_fetch_layer_uri_re_raises_client_error(self): + lambda_client_mock = Mock() + lambda_client_mock.get_layer_version.side_effect = ClientError( + error_response={'Error': {'Code': 'Unknown'}}, operation_name="lambda") + download_layers = LayerDownloader("/", ".", lambda_client_mock) + + layer = Mock() + layer.layer_arn = "arn" + layer.version = 1 + + with self.assertRaises(ClientError): + download_layers._fetch_layer_uri(layer=layer)