diff --git a/localstack/services/cloudformation/cloudformation_starter.py b/localstack/services/cloudformation/cloudformation_starter.py index b0e1c7db3b940..5040345035d3b 100644 --- a/localstack/services/cloudformation/cloudformation_starter.py +++ b/localstack/services/cloudformation/cloudformation_starter.py @@ -14,7 +14,7 @@ from localstack import config from localstack.constants import DEFAULT_PORT_CLOUDFORMATION_BACKEND, DEFAULT_REGION from localstack.utils.aws import aws_stack -from localstack.utils.common import short_uid +from localstack.utils.common import short_uid, FuncThread from localstack.stepfunctions import models as sfn_models from localstack.services.infra import ( get_service_protocol, start_proxy_for_service, do_run, canonicalize_api_names) @@ -27,15 +27,24 @@ # Maps (stack_name,resource_logical_id) -> Bool to indicate which resources are currently being updated CURRENTLY_UPDATING_RESOURCES = {} +# whether to start the API in a separate process +RUN_SERVER_IN_PROCESS = False + def start_cloudformation(port=None, asynchronous=False, update_listener=None): port = port or config.PORT_CLOUDFORMATION backend_port = DEFAULT_PORT_CLOUDFORMATION_BACKEND - cmd = 'python "%s" cloudformation -p %s -H 0.0.0.0' % (__file__, backend_port) print('Starting mock CloudFormation (%s port %s)...' % (get_service_protocol(), port)) - start_proxy_for_service('dynamodb', port, backend_port, update_listener) - env_vars = {'PYTHONPATH': ':'.join(sys.path)} - return do_run(cmd, asynchronous, env_vars=env_vars) + start_proxy_for_service('cloudformation', port, backend_port, update_listener) + if RUN_SERVER_IN_PROCESS: + cmd = 'python "%s" cloudformation -p %s -H 0.0.0.0' % (__file__, backend_port) + env_vars = {'PYTHONPATH': ':'.join(sys.path)} + return do_run(cmd, asynchronous, env_vars=env_vars) + else: + argv = ['cloudformation', '-p', str(backend_port), '-H', '0.0.0.0'] + thread = FuncThread(start_up, argv) + thread.start() + return thread def apply_patches(): @@ -120,6 +129,10 @@ def _parse_and_create_resource(logical_id, resource_json, resources_map, region_ if not resource: # create resource definition and store CloudFormation metadata in moto resource = parse_and_create_resource_orig(logical_id, resource_json, resources_map, region_name) + # Fix for moto which sometimes hard-codes region name as 'us-east-1' + if hasattr(resource, 'region_name') and resource.region_name != region_name: + LOG.debug('Updating incorrect region from %s to %s' % (resource.region_name, region_name)) + resource.region_name = region_name # Apply some fixes/patches to the resource names, then deploy resource in LocalStack update_resource_name(resource, resource_json) @@ -148,7 +161,7 @@ def find_id(resource): new_res_id = find_id(result) LOG.debug('Updating resource id: %s - %s, %s - %s' % (existing_id, new_res_id, resource, resource_json)) if new_res_id: - LOG.info('Updating resource ID from %s to %s' % (existing_id, new_res_id)) + LOG.info('Updating resource ID from %s to %s (%s)' % (existing_id, new_res_id, region_name)) update_resource_id(resource, new_res_id, props, region_name) else: LOG.warning('Unable to extract id for resource %s: %s' % (logical_id, result)) @@ -431,20 +444,24 @@ def create_backend_app(service): moto_server.create_backend_app = create_backend_app -def main(): - setup_logging() - +def start_up(*args): # patch moto implementation apply_patches() # add memory profiling endpoint inject_stats_endpoint() + return moto_main(*args) + + +def main(): + setup_logging() + # make sure all API names and ports are mapped properly canonicalize_api_names() # start API - sys.exit(moto_main()) + sys.exit(start_up()) if __name__ == '__main__': diff --git a/localstack/utils/aws/aws_stack.py b/localstack/utils/aws/aws_stack.py index 533ab9cda3e8b..46a1bda57bd7c 100644 --- a/localstack/utils/aws/aws_stack.py +++ b/localstack/utils/aws/aws_stack.py @@ -178,11 +178,12 @@ def connect_to_service(service_name, client=True, env=None, region_name=None, en """ Generic method to obtain an AWS service client using boto3, based on environment, region, or custom endpoint_url. """ - key_elements = [service_name, client, env, region_name, endpoint_url, config] + env = get_environment(env, region_name=region_name) + region = env.region if env.region != REGION_LOCAL else get_local_region() + key_elements = [service_name, client, env, region, endpoint_url, config] cache_key = '/'.join([str(k) for k in key_elements]) if cache_key not in BOTO_CLIENTS_CACHE: # Cache clients, as this is a relatively expensive operation - env = get_environment(env, region_name=region_name) my_session = get_boto3_session() method = my_session.client if client else my_session.resource verify = True @@ -190,7 +191,6 @@ def connect_to_service(service_name, client=True, env=None, region_name=None, en if is_local_env(env): endpoint_url = get_local_service_url(service_name) verify = False - region = env.region if env.region != REGION_LOCAL else get_local_region() BOTO_CLIENTS_CACHE[cache_key] = method(service_name, region_name=region, endpoint_url=endpoint_url, verify=verify, config=config) diff --git a/localstack/utils/server/multiserver.py b/localstack/utils/server/multiserver.py index 21e548795b7bc..abf2e6b246900 100644 --- a/localstack/utils/server/multiserver.py +++ b/localstack/utils/server/multiserver.py @@ -22,6 +22,9 @@ # API paths API_PATH_SERVERS = '/servers' +# whether to start the multiserver in a separate process +RUN_SERVER_IN_PROCESS = False + def start_api_server_locally(request): api = request.get('api') @@ -41,7 +44,7 @@ def thread_func(params): return result -def start_server(port): +def start_server(port, asynchronous=False): class ConfigListener(ProxyListener): def forward_request(self, method, path, data, **kwargs): @@ -62,6 +65,8 @@ def forward_request(self, method, path, data, **kwargs): proxy = GenericProxy(port, update_listener=ConfigListener()) proxy.start() + if asynchronous: + return proxy proxy.join() @@ -86,13 +91,17 @@ def start_server_process(port): port = port or MULTI_SERVER_PORT API_SERVERS['__server__'] = config = {'port': port} LOG.info('Starting multi API server process on port %s' % port) - cmd = '"%s" "%s" %s' % (sys.executable, __file__, port) - env_vars = { - 'PYTHONPATH': '.:%s' % constants.LOCALSTACK_ROOT_FOLDER - } - thread = ShellCommandThread(cmd, outfile=subprocess.PIPE, env_vars=env_vars, - inherit_cwd=True) - thread.start() + if RUN_SERVER_IN_PROCESS: + cmd = '"%s" "%s" %s' % (sys.executable, __file__, port) + env_vars = { + 'PYTHONPATH': '.:%s' % constants.LOCALSTACK_ROOT_FOLDER + } + thread = ShellCommandThread(cmd, outfile=subprocess.PIPE, env_vars=env_vars, + inherit_cwd=True) + thread.start() + else: + thread = start_server(port, asynchronous=True) + TMP_THREADS.append(thread) config['thread'] = thread wait_for_port_open(port, retries=20, sleep_time=1)