Merge b07e86f into 39d64ca

Dockerfile

@@ -3,6 +3,9 @@ FROM localstack/java-maven-node-python
 MAINTAINER Waldemar Hummer (waldemar.hummer@gmail.com)
 LABEL authors="Waldemar Hummer (waldemar.hummer@gmail.com), Gianluca Bortoli (giallogiallo93@gmail.com)"
 
+# install basic tools
+RUN pip install awscli awscli-local --upgrade
+
 # add files required to run "make install"
 ADD Makefile requirements.txt ./
 RUN mkdir -p localstack/utils/kinesis/ && mkdir -p localstack/services/ && \
@@ -23,13 +26,29 @@ ADD localstack/services/__init__.py localstack/services/install.py localstack/se
 # initialize installation (downloads remaining dependencies)
 RUN make init
 
+# (re-)install web dashboard dependencies (already installed in base image)
+ADD localstack/dashboard/web localstack/dashboard/web
+RUN make install-web
+
+# install supervisor config file and entrypoint script
+ADD bin/supervisord.conf /etc/supervisord.conf
+ADD bin/docker-entrypoint.sh /usr/local/bin/
+
+# expose service & web dashboard ports
+EXPOSE 4567-4584 8080
+
+# define command at startup
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+# expose default environment (required for aws-cli to work)
+ENV MAVEN_CONFIG=/opt/code/localstack \
+    USER=localstack \
+    PYTHONUNBUFFERED=1
+
 # add rest of the code
 ADD localstack/ localstack/
 ADD bin/localstack bin/localstack
 
-# (re-)install web dashboard dependencies (already installed in base image)
-RUN make install-web
-
 # fix some permissions and create local user
 RUN mkdir -p /.npm && \
     mkdir -p localstack/infra/elasticsearch/data && \
@@ -45,24 +64,6 @@ RUN mkdir -p /.npm && \
     adduser -D localstack && \
     ln -s `pwd` /tmp/localstack_install_dir
 
-# expose default environment (required for aws-cli to work)
-ENV MAVEN_CONFIG=/opt/code/localstack \
-    USER=localstack \
-    PYTHONUNBUFFERED=1
-
-# expose service & web dashboard ports
-EXPOSE 4567-4584 8080
-
-# install supervisor daemon & copy config file
-ADD bin/supervisord.conf /etc/supervisord.conf
-
-RUN pip install awscli awscli-local --upgrade 
-
-ADD bin/docker-entrypoint.sh /usr/local/bin/
-
-# define command at startup
-ENTRYPOINT ["docker-entrypoint.sh"]
-
 # run tests (to verify the build before pushing the image)
 ADD tests/ tests/
 RUN make test

bin/docker-entrypoint.sh

@@ -6,8 +6,8 @@ shopt -s nullglob
 supervisord -c /etc/supervisord.conf &
 
 until grep -q "^Ready.$" /tmp/localstack_infra.log >/dev/null 2>&1 ; do
-  echo "waiting for localstack"
-  sleep 2
+  echo "Waiting for all LocalStack services to be ready"
+  sleep 7
 done
 
 for f in /docker-entrypoint-initaws.d/*; do

bin/supervisord.conf

@@ -1,5 +1,6 @@
 [supervisord]
 nodaemon=true
+user=root
 logfile=/tmp/supervisord.log
 
 [program:infra]

docker-compose.yml

@@ -4,7 +4,7 @@ services:
   localstack:
     image: localstack/localstack
     ports:
-      - "4567-4584:4567-4584"
+      - "4567-4593:4567-4593"
       - "${PORT_WEB_UI-8080}:${PORT_WEB_UI-8080}"
     environment:
       - SERVICES=${SERVICES- }

localstack/plugins.py

@@ -5,7 +5,7 @@
 from localstack.services.infra import (register_plugin, Plugin,
     start_s3, start_sns, start_ses, start_apigateway, start_elasticsearch_service, start_lambda,
     start_redshift, start_firehose, start_cloudwatch, start_dynamodbstreams, start_route53,
-    start_ssm, start_sts, start_secretsmanager)
+    start_ssm, start_sts, start_secretsmanager, start_iam)
 from localstack.services.kinesis import kinesis_listener, kinesis_starter
 from localstack.services.dynamodb import dynamodb_listener, dynamodb_starter
 from localstack.services.apigateway import apigateway_listener
@@ -38,6 +38,8 @@ def register_localstack_plugins():
             start=start_ssm))
         register_plugin(Plugin('sts',
             start=start_sts))
+        register_plugin(Plugin('iam',
+            start=start_iam))
         register_plugin(Plugin('secretsmanager',
             start=start_secretsmanager))
         register_plugin(Plugin('apigateway',

localstack/services/awslambda/lambda_api.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import re
 import os
 import imp
@@ -14,7 +12,6 @@
 import traceback
 from io import BytesIO
 from datetime import datetime
-from six import iteritems
 from six.moves import cStringIO as StringIO
 from flask import Flask, Response, jsonify, request, make_response
 from localstack import config
@@ -306,7 +303,7 @@ def run_lambda(event, context, func_arn, version=None, suppress_output=False, as
         result, log_output = LAMBDA_EXECUTOR.execute(func_arn, func_details,
             event, context=context, version=version, asynchronous=asynchronous)
     except Exception as e:
-        return error_response('Error executing Lambda function: %s %s' % (e, traceback.format_exc()))
+        return error_response('Error executing Lambda function %s: %s %s' % (func_arn, e, traceback.format_exc()))
     finally:
         if suppress_output:
             sys.stdout = stdout_
@@ -516,7 +513,7 @@ def generic_handler(event, context):
 
 def do_list_functions():
     funcs = []
-    for f_arn, func in iteritems(arn_to_lambda):
+    for f_arn, func in arn_to_lambda.items():
         func_name = f_arn.split(':function:')[-1]
         arn = func_arn(func_name)
         func_details = arn_to_lambda.get(arn)

localstack/services/cloudformation/cloudformation_listener.py

@@ -7,7 +7,7 @@
 from localstack.services.generic_proxy import ProxyListener
 
 XMLNS_CLOUDFORMATION = 'http://cloudformation.amazonaws.com/doc/2010-05-15/'
-LOGGER = logging.getLogger(__name__)
+LOG = logging.getLogger(__name__)
 
 
 def error_response(message, code=400, error_type='ValidationError'):
@@ -39,7 +39,7 @@ def make_response(operation_name, content='', code=200):
 
 
 def validate_template(req_data):
-    LOGGER.debug(req_data)
+    LOG.debug('Validate CloudFormation template: %s' % req_data)
     response_content = """
         <Capabilities></Capabilities>
         <CapabilitiesReason></CapabilitiesReason>
@@ -71,6 +71,11 @@ def forward_request(self, method, path, data, headers):
 
         return True
 
+    def return_response(self, method, path, data, headers, response):
+        if response.status_code >= 400:
+            LOG.warning('Error response from CloudFormation (%s) %s %s: %s' %
+                        (response.status_code, method, path, response.content))
+
 
 # instantiate listener
 UPDATE_CLOUDFORMATION = ProxyListenerCloudFormation()

localstack/services/cloudformation/cloudformation_starter.py

@@ -1,13 +1,14 @@
 import sys
 import logging
 from moto.s3 import models as s3_models
+from moto.iam import models as iam_models
 from moto.core import BaseModel
 from moto.server import main as moto_main
 from moto.dynamodb import models as dynamodb_models
 from moto.apigateway import models as apigw_models
-from moto.cloudformation import parsing
+from moto.cloudformation import parsing, responses
 from boto.cloudformation.stack import Output
-from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
+from moto.cloudformation.exceptions import ValidationError, UnformattedGetAttTemplateException
 from localstack.config import PORT_CLOUDFORMATION
 from localstack.utils.aws import aws_stack
 from localstack.utils.common import short_uid
@@ -19,6 +20,9 @@
 
 LOG = logging.getLogger(__name__)
 
+# Maps (stack_name,resource_logical_id) -> Bool to indicate which resources are currently being updated
+CURRENTLY_UPDATING_RESOURCES = {}
+
 
 def start_cloudformation(port=PORT_CLOUDFORMATION, asynchronous=False, update_listener=None):
     backend_port = DEFAULT_PORT_CLOUDFORMATION_BACKEND
@@ -61,6 +65,15 @@ def clean_json(resource_json, resources_map):
     # Patch parse_and_create_resource method in moto to deploy resources in LocalStack
 
     def parse_and_create_resource(logical_id, resource_json, resources_map, region_name):
+        stack_name = resources_map.get('AWS::StackName')
+        resource_hash_key = (stack_name, logical_id)
+
+        # If the current stack is being updated, avoid
+        updating = CURRENTLY_UPDATING_RESOURCES.get(resource_hash_key)
+        LOG.debug('Currently updating stack resource %s/%s: %s' % (stack_name, logical_id, updating))
+        if updating:
+            return None
+
         # parse and get final resource JSON
         resource_tuple = parsing.parse_resource(logical_id, resource_json, resources_map)
         if not resource_tuple:
@@ -71,7 +84,6 @@ def parse_and_create_resource(logical_id, resource_json, resources_map, region_n
         resource = parse_and_create_resource_orig(logical_id, resource_json, resources_map, region_name)
 
         # check whether this resource needs to be deployed
-        stack_name = resources_map.get('AWS::StackName')
         resource_wrapped = {logical_id: resource_json}
         should_be_deployed = template_deployer.should_be_deployed(logical_id, resource_wrapped, stack_name)
         if not should_be_deployed:
@@ -80,7 +92,12 @@ def parse_and_create_resource(logical_id, resource_json, resources_map, region_n
 
         # deploy resource in LocalStack
         LOG.debug('Deploying CloudFormation resource: %s' % resource_json)
-        result = template_deployer.deploy_resource(logical_id, resource_wrapped, stack_name=stack_name)
+
+        try:
+            CURRENTLY_UPDATING_RESOURCES[resource_hash_key] = True
+            result = template_deployer.deploy_resource(logical_id, resource_wrapped, stack_name=stack_name)
+        finally:
+            CURRENTLY_UPDATING_RESOURCES[resource_hash_key] = False
         props = resource_json.get('Properties') or {}
 
         # update id in created resource
@@ -163,18 +180,31 @@ def parse_output(output_logical_id, output_json, resources_map):
     parse_output_orig = parsing.parse_output
     parsing.parse_output = parse_output
 
-    # Patch DynamoDB get_cfn_attribute(..) method to fix a bug in moto
+    # Patch DynamoDB get_cfn_attribute(..) method in moto
 
-    def get_cfn_attribute(self, attribute_name):
+    def DynamoDB_Table_get_cfn_attribute(self, attribute_name):
         try:
-            return get_cfn_attribute_orig(self, attribute_name)
+            return DynamoDB_Table_get_cfn_attribute_orig(self, attribute_name)
         except Exception:
             if attribute_name == 'Arn':
                 return aws_stack.dynamodb_table_arn(table_name=self.name)
             raise
 
-    get_cfn_attribute_orig = dynamodb_models.Table.get_cfn_attribute
-    dynamodb_models.Table.get_cfn_attribute = get_cfn_attribute
+    DynamoDB_Table_get_cfn_attribute_orig = dynamodb_models.Table.get_cfn_attribute
+    dynamodb_models.Table.get_cfn_attribute = DynamoDB_Table_get_cfn_attribute
+
+    # Patch IAM get_cfn_attribute(..) method in moto
+
+    def IAM_Role_get_cfn_attribute(self, attribute_name):
+        try:
+            return IAM_Role_get_cfn_attribute_orig(self, attribute_name)
+        except Exception:
+            if attribute_name == 'Arn':
+                return aws_stack.role_arn(self.name)
+            raise
+
+    IAM_Role_get_cfn_attribute_orig = iam_models.Role.get_cfn_attribute
+    iam_models.Role.get_cfn_attribute = IAM_Role_get_cfn_attribute
 
     # add CloudWatch types
 
@@ -241,6 +271,28 @@ def Method_create_from_cloudformation_json(cls, resource_name, cloudformation_js
     apigw_models.Method.create_from_cloudformation_json = Method_create_from_cloudformation_json
     # TODO: add support for AWS::ApiGateway::Model, AWS::ApiGateway::RequestValidator, ...
 
+    # fix AttributeError in moto's CloudFormation describe_stack_resource
+
+    def describe_stack_resource(self):
+        stack_name = self._get_param('StackName')
+        stack = self.cloudformation_backend.get_stack(stack_name)
+        logical_resource_id = self._get_param('LogicalResourceId')
+
+        for stack_resource in stack.stack_resources:
+            # Note: Line below has been patched
+            # if stack_resource.logical_resource_id == logical_resource_id:
+            if stack_resource and stack_resource.logical_resource_id == logical_resource_id:
+                resource = stack_resource
+                break
+        else:
+            raise ValidationError(logical_resource_id)
+
+        template = self.response_template(
+            responses.DESCRIBE_STACK_RESOURCE_RESPONSE_TEMPLATE)
+        return template.render(stack=stack, resource=resource)
+
+    responses.CloudFormationResponse.describe_stack_resource = describe_stack_resource
+
 
 def main():
     setup_logging()

localstack/services/infra.py

@@ -17,7 +17,7 @@
 from localstack.config import (USE_SSL, PORT_ROUTE53, PORT_S3,
     PORT_FIREHOSE, PORT_LAMBDA, PORT_SNS, PORT_REDSHIFT, PORT_CLOUDWATCH,
     PORT_DYNAMODBSTREAMS, PORT_SES, PORT_ES, PORT_APIGATEWAY, PORT_SSM,
-    PORT_SECRETSMANAGER, PORT_STS)
+    PORT_SECRETSMANAGER, PORT_STS, PORT_IAM)
 from localstack.utils import common, persistence
 from localstack.utils.common import (run, TMP_THREADS, in_ci, run_cmd_safe,
     TIMESTAMP_FORMAT, FuncThread, ShellCommandThread, mkdir)
@@ -161,6 +161,10 @@ def start_sts(port=PORT_STS, asynchronous=False):
     return start_moto_server('sts', port, name='STS', asynchronous=asynchronous)
 
 
+def start_iam(port=PORT_IAM, asynchronous=False):
+    return start_moto_server('iam', port, name='IAM', asynchronous=asynchronous)
+
+
 def start_redshift(port=PORT_REDSHIFT, asynchronous=False):
     return start_moto_server('redshift', port, name='Redshift', asynchronous=asynchronous)
 
@@ -208,6 +212,7 @@ def setup_logging():
     # disable some logs and warnings
     warnings.filterwarnings('ignore')
     logging.captureWarnings(True)
+    logging.getLogger('boto3').setLevel(logging.INFO)
     logging.getLogger('s3transfer').setLevel(logging.INFO)
     logging.getLogger('docker').setLevel(logging.WARNING)
     logging.getLogger('urllib3').setLevel(logging.WARNING)

localstack/services/stepfunctions/stepfunctions_listener.py

@@ -0,0 +1,15 @@
+import logging
+from localstack.services.generic_proxy import ProxyListener
+
+LOG = logging.getLogger(__name__)
+
+
+class ProxyListenerStepFunctions(ProxyListener):
+
+    def forward_request(self, method, path, data, headers):
+        LOG.debug('StepFunctions request:', method, path, data)
+        return True
+
+
+# instantiate listener
+UPDATE_STEPFUNCTIONS = ProxyListenerStepFunctions()