Add CloudFoundry integration for Step Functions and API Gateway

localstack/services/apigateway/apigateway_listener.py

@@ -2,7 +2,6 @@
 import logging
 import json
 import requests
-
 from requests.models import Response
 from flask import Response as FlaskResponse
 from localstack.constants import APPLICATION_JSON, PATH_USER_REQUEST
@@ -149,6 +148,16 @@ def forward_request(self, method, path, data, headers):
 
         return True
 
+    def return_response(self, method, path, data, headers, response):
+        # fix backend issue (missing support for API documentation)
+        if re.match(r'/restapis/[^/]+/documentation/versions', path):
+            if response.status_code == 404:
+                response = Response()
+                response.status_code = 200
+                result = {'position': '1', 'items': []}
+                response._content = json.dumps(result)
+                return response
+
 
 # instantiate listener
 UPDATE_APIGATEWAY = ProxyListenerApiGateway()

localstack/services/awslambda/lambda_api.py

@@ -34,7 +34,7 @@
     LAMBDA_RUNTIME_RUBY25)
 from localstack.utils.common import (to_str, load_file, save_file, TMP_FILES, ensure_readable,
     mkdir, unzip, is_zip_file, run, short_uid, is_jar_archive, timestamp, TIMESTAMP_FORMAT_MILLIS,
-    md5, new_tmp_file)
+    md5, new_tmp_file, parse_chunked_data)
 from localstack.utils.aws import aws_stack, aws_responses
 from localstack.utils.analytics import event_publisher
 from localstack.utils.cloudwatch.cloudwatch_util import cloudwatched
@@ -548,6 +548,14 @@ def format_func_details(func_details, version=None, always_add_version=False):
 # ------------
 
 
+@app.before_request
+def before_request():
+    # fix to enable chunked encoding, as this is used by some Lambda clients
+    transfer_encoding = request.headers.get('Transfer-Encoding', None)
+    if transfer_encoding == 'chunked':
+        request.environ['wsgi.input_terminated'] = True
+
+
 @app.route('%s/functions' % PATH_ROOT, methods=['POST'])
 def create_function():
     """ Create new function
@@ -771,12 +779,18 @@ def invoke_function(function):
     if qualifier and not arn_to_lambda.get(arn).qualifier_exists(qualifier):
         return error_response('Function does not exist: {0}:{1}'.format(arn, qualifier), 404,
                               error_type='ResourceNotFoundException')
-    data = None
-    if request.data:
+    data = request.get_data()
+    if data:
+        data = to_str(data)
         try:
-            data = json.loads(to_str(request.data))
+            data = json.loads(data)
         except Exception:
-            return error_response('The payload is not JSON', 415, error_type='UnsupportedMediaTypeException')
+            try:
+                # try to read chunked content
+                data = json.loads(parse_chunked_data(data))
+            except Exception:
+                return error_response('The payload is not JSON: %s' % data, 415,
+                                      error_type='UnsupportedMediaTypeException')
 
     # Default invocation type is RequestResponse
     invocation_type = request.environ.get('HTTP_X_AMZ_INVOCATION_TYPE', 'RequestResponse')

localstack/services/cloudformation/cloudformation_listener.py

@@ -2,18 +2,13 @@
 import logging
 from requests.models import Response
 from six.moves.urllib import parse as urlparse
-from localstack.constants import DEFAULT_REGION, TEST_AWS_ACCOUNT_ID
 from localstack.utils.common import to_str
-from localstack.utils.aws import aws_stack
 from localstack.utils.cloudformation import template_deployer
 from localstack.services.generic_proxy import ProxyListener
 
 XMLNS_CLOUDFORMATION = 'http://cloudformation.amazonaws.com/doc/2010-05-15/'
 LOGGER = logging.getLogger(__name__)
 
-# maps change set names to change set details
-CHANGE_SETS = {}
-
 
 def error_response(message, code=400, error_type='ValidationError'):
     response = Response()
@@ -43,84 +38,6 @@ def make_response(operation_name, content='', code=200):
     return response
 
 
-def stack_exists(stack_name):
-    cloudformation = aws_stack.connect_to_service('cloudformation')
-    stacks = cloudformation.list_stacks()
-    for stack in stacks['StackSummaries']:
-        if stack['StackName'] == stack_name:
-            return True
-    return False
-
-
-# TODO - deprecated - remove!
-def create_stack(req_data):
-    stack_name = req_data.get('StackName')[0]
-    if stack_exists(stack_name):
-        message = 'The resource with the name requested already exists.'
-        return error_response(message, error_type='AlreadyExists')
-    # create stack
-    cloudformation_service = aws_stack.connect_to_service('cloudformation')
-    template = template_deployer.template_to_json(req_data.get('TemplateBody')[0])
-    cloudformation_service.create_stack(StackName=stack_name,
-        TemplateBody=template)
-    # now run the actual deployment
-    template_deployer.deploy_template(template, stack_name)
-    return True
-
-
-# TODO - deprecated - remove!
-def create_change_set(req_data):
-    cs_name = req_data.get('ChangeSetName')[0]
-    change_set_uuid = uuid.uuid4()
-    cs_arn = 'arn:aws:cloudformation:%s:%s:changeSet/%s/%s' % (
-        DEFAULT_REGION, TEST_AWS_ACCOUNT_ID, cs_name, change_set_uuid)
-    CHANGE_SETS[cs_arn] = dict(req_data)
-    response = make_response('CreateChangeSet', '<Id>%s</Id>' % cs_arn)
-    return response
-
-
-# TODO - deprecated - remove!
-def describe_change_set(req_data):
-    cs_arn = req_data.get('ChangeSetName')[0]
-    cs_details = CHANGE_SETS.get(cs_arn)
-    if not cs_details:
-        return error_response('Change Set %s does not exist' % cs_arn, 404, 'ChangeSetNotFound')
-    stack_name = cs_details.get('StackName')[0]
-    response_content = """
-        <StackName>%s</StackName>
-        <ChangeSetId>%s</ChangeSetId>
-        <Status>CREATE_COMPLETE</Status>""" % (stack_name, cs_arn)
-    response = make_response('DescribeChangeSet', response_content)
-    return response
-
-
-# TODO - deprecated - remove!
-def execute_change_set(req_data):
-    cs_arn = req_data.get('ChangeSetName')[0]
-    stack_name = req_data.get('StackName')[0]
-    cs_details = CHANGE_SETS.get(cs_arn)
-    if not cs_details:
-        return error_response('Change Set %s does not exist' % cs_arn, 404, 'ChangeSetNotFound')
-
-    # convert to JSON (might have been YAML, and update_stack/create_stack seem to only work with JSON)
-    template = template_deployer.template_to_json(cs_details.get('TemplateBody')[0])
-
-    # update stack information
-    cloudformation_service = aws_stack.connect_to_service('cloudformation')
-    if stack_exists(stack_name):
-        cloudformation_service.update_stack(StackName=stack_name,
-            TemplateBody=template)
-    else:
-        cloudformation_service.create_stack(StackName=stack_name,
-            TemplateBody=template)
-
-    # now run the actual deployment
-    template_deployer.deploy_template(template, stack_name)
-
-    response = make_response('ExecuteChangeSet')
-    return response
-
-
 def validate_template(req_data):
     LOGGER.debug(req_data)
     response_content = """
@@ -131,7 +48,6 @@ def validate_template(req_data):
         <Parameters>
         </Parameters>
     """
-
     try:
         template_deployer.template_to_json(req_data.get('TemplateBody')[0])
         response = make_response('ValidateTemplate', response_content)

localstack/services/cloudformation/cloudformation_starter.py

@@ -1,14 +1,20 @@
 import sys
 import logging
 from moto.s3 import models as s3_models
+from moto.core import BaseModel
 from moto.server import main as moto_main
 from moto.dynamodb import models as dynamodb_models
+from moto.apigateway import models as apigw_models
 from moto.cloudformation import parsing
 from boto.cloudformation.stack import Output
+from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
 from localstack.config import PORT_CLOUDFORMATION
-from localstack.constants import DEFAULT_PORT_CLOUDFORMATION_BACKEND
-from localstack.services.infra import get_service_protocol, start_proxy_for_service, do_run
 from localstack.utils.aws import aws_stack
+from localstack.utils.common import short_uid
+from localstack.constants import DEFAULT_PORT_CLOUDFORMATION_BACKEND, DEFAULT_REGION
+from localstack.stepfunctions import models as sfn_models
+from localstack.services.infra import (
+    get_service_protocol, start_proxy_for_service, do_run, setup_logging)
 from localstack.utils.cloudformation import template_deployer
 
 LOG = logging.getLogger(__name__)
@@ -36,6 +42,22 @@ def get_key(self, bucket_name, key_name, version_id=None):
 
     s3_models.S3Backend.get_key = get_key
 
+    # Patch clean_json in moto
+
+    def clean_json(resource_json, resources_map):
+        result = clean_json_orig(resource_json, resources_map)
+        if isinstance(result, BaseModel):
+            if isinstance(resource_json, dict) and 'Ref' in resource_json:
+                if hasattr(result, 'id'):
+                    return result.id
+                if hasattr(result, 'name'):
+                    # TODO: Check if this is the desired behavior. Better return ARN instead of ID?
+                    return result.name
+        return result
+
+    clean_json_orig = parsing.clean_json
+    parsing.clean_json = clean_json
+
     # Patch parse_and_create_resource method in moto to deploy resources in LocalStack
 
     def parse_and_create_resource(logical_id, resource_json, resources_map, region_name):
@@ -48,12 +70,79 @@ def parse_and_create_resource(logical_id, resource_json, resources_map, region_n
         # create resource definition and store CloudFormation metadata in moto
         resource = parse_and_create_resource_orig(logical_id, resource_json, resources_map, region_name)
 
-        # deploy resource in LocalStack
+        # check whether this resource needs to be deployed
         stack_name = resources_map.get('AWS::StackName')
         resource_wrapped = {logical_id: resource_json}
-        if template_deployer.should_be_deployed(logical_id, resource_wrapped, stack_name):
-            LOG.debug('Deploying CloudFormation resource: %s' % resource_json)
-            template_deployer.deploy_resource(logical_id, resource_wrapped, stack_name=stack_name)
+        should_be_deployed = template_deployer.should_be_deployed(logical_id, resource_wrapped, stack_name)
+        if not should_be_deployed:
+            LOG.debug('Resource %s need not be deployed: %s' % (logical_id, resource_json))
+            return resource
+
+        # deploy resource in LocalStack
+        LOG.debug('Deploying CloudFormation resource: %s' % resource_json)
+        result = template_deployer.deploy_resource(logical_id, resource_wrapped, stack_name=stack_name)
+        props = resource_json.get('Properties') or {}
+
+        # update id in created resource
+        def find_id(result):
+            for id_attr in ('Id', 'id', 'ResourceId', 'RestApiId', 'DeploymentId'):
+                if id_attr in result:
+                    return result[id_attr]
+
+        def update_id(resource, new_id):
+            # Update the ID of the given resource.
+            # NOTE: this is a bit of a hack, which is required because
+            # of the order of events when CloudFormation resources are created.
+            # When we process a request to create a CF resource that's part of a
+            # stack, say, an API Gateway Resource, then we (1) create the object
+            # in memory in moto, which generates a random ID for the resource, and
+            # (2) create the actual resource in the backend service using
+            # template_deployer.deploy_resource(..) (see above).
+            # The resource created in (2) now has a different ID than the resource
+            # created in (1), which leads to downstream problems. Hence, we need
+            # the logic below to reconcile the ids, i.e., apply IDs from (2) to (1).
+
+            backend = apigw_models.apigateway_backends[region_name]
+            if isinstance(resource, apigw_models.RestAPI):
+                backend.apis.pop(resource.id, None)
+                backend.apis[new_id] = resource
+                # We also need to fetch the resources to replace the root resource
+                # that moto automatically adds to newly created RestAPI objects
+                client = aws_stack.connect_to_service('apigateway')
+                resources = client.get_resources(restApiId=new_id, limit=500)['items']
+                # make sure no resources have been added in addition to the root /
+                assert len(resource.resources) == 1
+                resource.resources = {}
+                for res in resources:
+                    res_path_part = res.get('pathPart') or res.get('path')
+                    child = resource.add_child(res_path_part, res.get('parentId'))
+                    resource.resources.pop(child.id)
+                    child.id = res['id']
+                    resource.resources[child.id] = child
+                resource.id = new_id
+            elif isinstance(resource, apigw_models.Resource):
+                api_id = props['RestApiId']
+                backend.apis[api_id].resources.pop(resource.id, None)
+                backend.apis[api_id].resources[new_id] = resource
+                resource.id = new_id
+            elif isinstance(resource, apigw_models.Deployment):
+                api_id = props['RestApiId']
+                backend.apis[api_id].deployments.pop(resource['id'], None)
+                backend.apis[api_id].deployments[new_id] = resource
+                resource['id'] = new_id
+            else:
+                LOG.warning('Unexpected resource type when updating ID: %s' % type(resource))
+
+        if hasattr(resource, 'id') or (isinstance(resource, dict) and resource.get('id')):
+            existing_id = resource.id if hasattr(resource, 'id') else resource['id']
+            new_res_id = find_id(result)
+            LOG.debug('Updating resource id: %s - %s, %s - %s' % (existing_id, new_res_id, resource, resource_json))
+            if new_res_id:
+                LOG.info('Updating resource ID from %s to %s' % (existing_id, new_res_id))
+                update_id(resource, new_res_id)
+            else:
+                LOG.warning('Unable to extract id for resource %s: %s' % (logical_id, result))
+
         return resource
 
     parse_and_create_resource_orig = parsing.parse_and_create_resource
@@ -87,8 +176,75 @@ def get_cfn_attribute(self, attribute_name):
     get_cfn_attribute_orig = dynamodb_models.Table.get_cfn_attribute
     dynamodb_models.Table.get_cfn_attribute = get_cfn_attribute
 
+    # add CloudWatch types
+
+    parsing.MODEL_MAP['AWS::ApiGateway::Deployment'] = apigw_models.Deployment
+    parsing.MODEL_MAP['AWS::ApiGateway::Method'] = apigw_models.Method
+    parsing.MODEL_MAP['AWS::ApiGateway::Resource'] = apigw_models.Resource
+    parsing.MODEL_MAP['AWS::ApiGateway::RestApi'] = apigw_models.RestAPI
+    parsing.MODEL_MAP['AWS::StepFunctions::StateMachine'] = sfn_models.StateMachine
+
+    @classmethod
+    def RestAPI_create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        props = cloudformation_json['Properties']
+        name = props['Name']
+        region_name = props.get('Region') or DEFAULT_REGION
+        description = props.get('Description') or ''
+        id = props.get('Id') or short_uid()
+        return apigw_models.RestAPI(id, region_name, name, description)
+
+    def RestAPI_get_cfn_attribute(self, attribute_name):
+        if attribute_name == 'Id':
+            return self.id
+        if attribute_name == 'Region':
+            return self.region_name
+        if attribute_name == 'Name':
+            return self.name
+        if attribute_name == 'Description':
+            return self.description
+        if attribute_name == 'RootResourceId':
+            for id, resource in self.resources.items():
+                if resource.parent_id is None:
+                    return resource.id
+            return None
+        raise UnformattedGetAttTemplateException()
+
+    @classmethod
+    def Deployment_create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        props = cloudformation_json['Properties']
+        name = props['StageName']
+        deployment_id = props.get('Id') or short_uid()
+        description = props.get('Description') or ''
+        return apigw_models.Deployment(deployment_id, name, description)
+
+    @classmethod
+    def Resource_create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        props = cloudformation_json['Properties']
+        region_name = props.get('Region') or DEFAULT_REGION
+        path_part = props.get('PathPart')
+        api_id = props.get('RestApiId')
+        parent_id = props.get('ParentId')
+        id = props.get('Id') or short_uid()
+        return apigw_models.Resource(id, region_name, api_id, path_part, parent_id)
+
+    @classmethod
+    def Method_create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        props = cloudformation_json['Properties']
+        method_type = props.get('HttpMethod')
+        authorization_type = props.get('AuthorizationType')
+        return apigw_models.Method(method_type, authorization_type)
+
+    apigw_models.RestAPI.create_from_cloudformation_json = RestAPI_create_from_cloudformation_json
+    apigw_models.RestAPI.get_cfn_attribute = RestAPI_get_cfn_attribute
+    apigw_models.Deployment.create_from_cloudformation_json = Deployment_create_from_cloudformation_json
+    apigw_models.Resource.create_from_cloudformation_json = Resource_create_from_cloudformation_json
+    apigw_models.Method.create_from_cloudformation_json = Method_create_from_cloudformation_json
+    # TODO: add support for AWS::ApiGateway::Model, AWS::ApiGateway::RequestValidator, ...
+
 
 def main():
+    setup_logging()
+
     # patch moto implementation
     apply_patches()