Lambda enhancements

   - Added support for identifying the docker network when running lambda functions in docker containers

    The docker network used when running lambda functions in a container may be specified with the LAMBDA_DEFAULT_DOCKER_NETWORK and LAMBDA_SUBNET_AS_DOCKERNET environment variables along with the subnet specified in the lambda's vpc config

   - Added environment variables specifying the function arn and function name
     for use in docker command options -- useful when combined with LAMBDA_DOCKER_OPTIONS
   - fixed issue where environment variables were not returned in get_function_configuration
   - Added support for custom docker actions when running lambda functions in docker containers
   custom options to be passed to docker can be specified via the LAMBDA_DOCKER_OPTIONS
   environment variables

   - Added test configurations for lambda docker network config
   - Added lambda networking test to travis build

.gitignore

@@ -25,3 +25,7 @@ localstack/infra/
 /.idea
 **/obj/**
 **/bin/**
+.vscode
+.idea
+tmp
+/.pydevproject

.travis.yml

@@ -8,10 +8,6 @@ services:
 python:
   - "3.6"
 
-branches:
-  only:
-    - master
-
 before_install:
   - "sudo apt-get purge openjdk-6*"
   - "sudo apt-get purge openjdk-7*"
@@ -37,18 +33,25 @@ install:
 script:
   - set -e # fail fast
   # run tests using Python 3
-  - DEBUG=1 LAMBDA_EXECUTOR=docker TEST_ERROR_INJECTION=1 make test
+  - DEBUG=1 LAMBDA_EXECUTOR=docker make test
+  - DEBUG=1 LAMBDA_EXECUTOR=docker make test-lambdanet
   - LAMBDA_EXECUTOR=local USE_SSL=1 make test
   # run tests using Python 2
   #   Note: we're not using multiple versions in the top-level "python" configuration,
   #   but instead reinstall using 2.x here, as that allows us to use some cached libs etc.
   - "make reinstall-p2 > /dev/null"
   - make init
   - DEBUG=1 LAMBDA_EXECUTOR=docker-reuse USE_SSL=1 make test
-  # build Docker image
-  - make docker-build
+  - DEBUG=1 LAMBDA_EXECUTOR=docker-reuse make test-lambdanet
+  - DEBUG=1 LAMBDA_EXECUTOR=docker make test
+  - DEBUG=1 LAMBDA_EXECUTOR=docker make test-lambdanet
+  - LAMBDA_EXECUTOR=local USE_SSL=1 make test
   # run Java tests
   - make test-java-if-changed
+
+  # build Docker image
+  - make docker-build
+
   # push Docker image (if on master branch)
   - make docker-push-master
 

Makefile

@@ -87,7 +87,40 @@ web:               ## Start web application (dashboard)
 
 test:              ## Run automated tests
 	make lint && \
-		($(VENV_RUN); DEBUG=$(DEBUG) PYTHONPATH=`pwd` nosetests --with-coverage --logging-level=WARNING --nocapture --no-skip --exe --cover-erase --cover-tests --cover-inclusive --cover-package=localstack --with-xunit --exclude='$(VENV_DIR).*' --ignore-files='lambda_python3.py' .)
+	($(VENV_RUN); DEBUG=$(DEBUG) PYTHONPATH=`pwd` nosetests --with-coverage --logging-level=WARNING --nocapture --no-skip --exe --cover-erase --cover-tests --cover-inclusive --cover-package=localstack --with-xunit --exclude='$(VENV_DIR).*' --ignore-files='lambda_python3.+' .)
+
+test-lambdanet:    ## Test running lambdas in specific docker networks
+
+	# setup docker infrastructure needed by tests
+	# basically run simple http servers in separate bridge networks
+	-docker network create -d bridge test_localstack_lambdanet_default
+	-docker network create -d bridge test_localstack_lambdanet_custom
+	-docker run -d -P --network=test_localstack_lambdanet_default \
+			  --name=test_localstack_lambdanet_default_id  \
+	          --net-alias=networkidentifier --rm \
+			  -v $$(pwd)/tests/integration/nwfiles/default:/www fnichol/uhttpd
+	-docker run -d -P --network=test_localstack_lambdanet_custom \
+			  --name=test_localstack_lambdanet_custom_id  \
+	          --net-alias=networkidentifier --rm \
+			  -v $$(pwd)/tests/integration/nwfiles/custom:/www fnichol/uhttpd
+	# run the tests -- note the lambda network settings
+	($(VENV_RUN); \
+	DEBUG=$(DEBUG) \
+	PYTHONPATH=`pwd` \
+	LAMBDA_EXECUTOR=docker \
+	LAMBDA_DEFAULT_DOCKER_NETWORK=test_localstack_lambdanet_default \
+	LAMBDA_SUBNET_AS_DOCKERNET=1 \
+	nosetests --with-coverage --logging-level=WARNING --nocapture \
+	          --no-skip --exe --cover-erase --cover-tests --cover-inclusive \
+			  --cover-package=localstack --with-xunit \
+			  --exclude='$(VENV_DIR).*' \
+			  --ignore-files='lambda_python3.+' .)
+	#cleanup our docker infrastructure
+	docker kill test_localstack_lambdanet_default_id
+	docker kill test_localstack_lambdanet_custom_id
+	docker network rm test_localstack_lambdanet_default
+	docker network rm test_localstack_lambdanet_custom
+
 
 test-java:         ## Run tests for Java/JUnit compatibility
 	cd localstack/ext/java; mvn -q test && USE_SSL=1 mvn -q test

README.md

@@ -163,6 +163,22 @@ You can pass the following environment variables to LocalStack:
     - `false`: your Lambda function definitions will be passed to the container by mounting a
       volume (potentially faster). This requires to have the Docker client and the Docker
       host on the same machine.
+* `LAMBDA_DEFAULT_DOCKER_NETWORK`:
+    - When set, and running lambda functions in a docker container 
+      (LAMBDA_EXECUTOR=docker or docker-reuse) the docker container running the lambda function
+      will be attached to the docker network identified by LAMBDA_DEFAULT_DOCKER_NETWORK
+* `LAMBDA_SUBNET_AS_DOCKERNET`:
+    - When set to 1, and lambda functions are run in a docker container
+      (LAMBDA_EXECUTOR=docker or docker-reuse) if a lambda function is created with a vpc config,
+      the subnet identified in the vpc config will be used as the identifier of the docker network for the lambda function's docker container 
+* `LAMBDA_DOCKER_OPTIONS`:
+    - Specifies custom options to be passed to the docker engine as part of the docker run or docker create command.  
+
+        For example, to log lambda function output to a syslog server listening on the local host at port 5601, you can set LAMBDA_DOCKER_OPTIONS to:
+
+          --log-driver=syslog --log-opt=tag=lambda.$LAMBDA_FUNCTION_NAME --log-opt=syslog-address=tcp://127.0.0.1:5601
+
+
 * `DATA_DIR`: Local directory for saving persistent data (currently only supported for these services:
   Kinesis, DynamoDB, Elasticsearch, S3). Set it to `/tmp/localstack/data` to enable persistence
   (`/tmp/localstack` is mounted into the Docker container), leave blank to disable
@@ -180,7 +196,11 @@ Additionally, the following *read-only* environment variables are available:
   (e.g., to store an item to DynamoDB or S3 from Lambda).
   The variable `LOCALSTACK_HOSTNAME` is available for both, local Lambda execution
   (`LAMBDA_EXECUTOR=local`) and execution inside separate Docker containers (`LAMBDA_EXECUTOR=docker`).
+* `LAMBDA_FUNCTION_ARN` is  arn of the function executed
+* `LAMBDA_FUNCTION_NAME` is the name of the function being created/executed
 
+These variables can be referenced from within a lambda function, or in the value of the `LAMBDA_DOCKER_OPTIONS` or `LAMBDA_DEFAULT_DOCKER_NETWORK` variables specified above.
+
 ## Accessing the infrastructure via CLI or code
 
 You can point your `aws` CLI to use the local infrastructure, for example:

localstack/config.py

@@ -60,6 +60,10 @@
     except Exception as e:
         pass
 
+LAMBDA_DEFAULT_DOCKER_NETWORK = os.environ.get('LAMBDA_DEFAULT_DOCKER_NETWORK', '').strip()
+LAMBDA_SUBNET_AS_DOCKERNET = os.environ.get('LAMBDA_SUBNET_AS_DOCKERNET', False)    
+LAMBDA_DOCKER_OPTIONS = ' ' + os.environ.get('LAMBDA_DOCKER_OPTIONS', '')
+
 # list of environment variable names used for configuration.
 # Make sure to keep this in sync with the above!
 # Note: do *not* include DATA_DIR in this list, as it is treated separately

localstack/services/awslambda/lambda_api.py

@@ -474,6 +474,7 @@ def format_func_details(func_details, version=None, always_add_version=False):
         'Handler': func_details.handler,
         'Runtime': func_details.runtime,
         'Timeout': func_details.timeout,
+        'VpcConfig': func_details.vpc_config,
         'Environment': func_details.envvars,
         # 'Description': ''
         # 'MemorySize': 192,
@@ -507,12 +508,15 @@ def create_function():
         if arn in arn_to_lambda:
             return error_response('Function already exist: %s' %
                 lambda_name, 409, error_type='ResourceConflictException')
+
         arn_to_lambda[arn] = func_details = LambdaFunction(arn)
         func_details.versions = {'$LATEST': {'CodeSize': 50}}
         func_details.handler = data['Handler']
         func_details.runtime = data['Runtime']
         func_details.envvars = data.get('Environment', {}).get('Variables', {})
         func_details.timeout = data.get('Timeout')
+        func_details.vpc_config = data.get('VpcConfig', {'SubnetIds': [], 'SecurityGroupIds': []})
+
         result = set_function_code(data['Code'], lambda_name)
         if isinstance(result, Response):
             del arn_to_lambda[arn]
@@ -648,6 +652,7 @@ def get_function_configuration(function):
         operationId: 'getFunctionConfiguration'
         parameters:
     """
+
     arn = func_arn(function)
     lambda_details = arn_to_lambda.get(arn)
     if not lambda_details:
@@ -680,6 +685,9 @@ def update_function_configuration(function):
         lambda_details.envvars = data.get('Environment', {}).get('Variables', {})
     if data.get('Timeout'):
         lambda_details.timeout = data['Timeout']
+    if data.get('VpcConfig'):
+        lambda_details.vpc_config = data['VpcConfig']
+
     result = {}
     return jsonify(result)
 

localstack/services/awslambda/lambda_executors.py

@@ -83,7 +83,7 @@ def __init__(self, name, entry_point):
 class LambdaExecutorContainers(LambdaExecutor):
     """ Abstract executor class for executing Lambda functions in Docker containers """
 
-    def prepare_execution(self, func_arn, env_vars, runtime, command, handler, lambda_cwd):
+    def prepare_execution(self, func_arn, func_details, env_vars, runtime, command, handler, lambda_cwd):
         raise Exception('Not implemented')
 
     def execute(self, func_arn, func_details, event, context=None, version=None, async=False):
@@ -110,6 +110,8 @@ def execute(self, func_arn, func_details, event, context=None, version=None, asy
         environment['AWS_LAMBDA_EVENT_BODY'] = event_body_escaped
         environment['HOSTNAME'] = docker_host
         environment['LOCALSTACK_HOSTNAME'] = docker_host
+        environment['LAMBDA_FUNCTION_ARN'] = func_arn
+        environment['LAMBDA_FUNCTION_NAME'] = func_arn.split(':function:')[-1]
 
         # custom command to execute in the container
         command = ''
@@ -125,14 +127,34 @@ def execute(self, func_arn, func_details, event, context=None, version=None, asy
                 (taskdir, LAMBDA_EXECUTOR_CLASS, handler, LAMBDA_EVENT_FILE))
 
         # determine the command to be executed (implemented by subclasses)
-        cmd = self.prepare_execution(func_arn, environment, runtime, command, handler, lambda_cwd)
+        cmd = self.prepare_execution(func_arn, func_details, environment, runtime, command, handler, lambda_cwd)
 
         # lambci writes the Lambda result to stdout and logs to stderr, fetch it from there!
         LOG.debug('Running lambda cmd: %s' % cmd)
         result, log_output = self.run_lambda_executor(cmd, environment, async)
         LOG.debug('Lambda result / log output:\n%s\n>%s' % (result.strip(), log_output.strip().replace('\n', '\n> ')))
         return result, log_output
 
+    def lambda_docker_network(self, func_arn, func_details):
+        network = None
+
+        if config.LAMBDA_SUBNET_AS_DOCKERNET and func_details.vpc_config:
+            subnets = func_details.vpc_config.get('SubnetIds', [])
+            if len(subnets) > 0:
+                network = subnets[0]
+
+        if not network and config.LAMBDA_DEFAULT_DOCKER_NETWORK:
+            network = config.LAMBDA_DEFAULT_DOCKER_NETWORK
+
+        return network
+
+    def lambda_docker_cmd_networksection(self, func_arn, func_details):
+        network = self.lambda_docker_network(func_arn, func_details)
+        if network:
+            return ' --network="%s" ' % network
+        else:
+            return ''
+
 
 class LambdaExecutorReuseContainers(LambdaExecutorContainers):
     """ Executor class for executing Lambda functions in re-usable Docker containers """
@@ -144,7 +166,7 @@ def __init__(self):
         # locking thread for creation/destruction of docker containers.
         self.docker_container_lock = threading.RLock()
 
-    def prepare_execution(self, func_arn, env_vars, runtime, command, handler, lambda_cwd):
+    def prepare_execution(self, func_arn, func_details, env_vars, runtime, command, handler, lambda_cwd):
 
         # check whether the Lambda has been invoked before
         has_been_invoked_before = func_arn in self.function_invoke_times
@@ -154,7 +176,7 @@ def prepare_execution(self, func_arn, env_vars, runtime, command, handler, lambd
 
         # create/verify the docker container is running.
         LOG.debug('Priming docker container with runtime "%s" and arn "%s".', runtime, func_arn)
-        container_info = self.prime_docker_container(runtime, func_arn, env_vars.items(), lambda_cwd)
+        container_info = self.prime_docker_container(runtime, func_arn, func_details, env_vars.items(), lambda_cwd)
 
         # Note: currently "docker exec" does not support --env-file, i.e., environment variables can only be
         # passed directly on the command line, using "-e" below. TODO: Update this code once --env-file is
@@ -196,7 +218,7 @@ def cleanup(self, arn=None):
         self.function_invoke_times = {}
         return self.destroy_existing_docker_containers()
 
-    def prime_docker_container(self, runtime, func_arn, env_vars, lambda_cwd):
+    def prime_docker_container(self, runtime, func_arn, func_details, env_vars, lambda_cwd):
         """
         Prepares a persistent docker container for a specific function.
         :param runtime: Lamda runtime environment. python2.7, nodejs6.10, etc.
@@ -219,6 +241,10 @@ def prime_docker_container(self, runtime, func_arn, env_vars, lambda_cwd):
 
                 env_vars_str = ' '.join(['-e {}={}'.format(k, cmd_quote(v)) for (k, v) in env_vars])
 
+                network = self.lambda_docker_cmd_networksection(func_arn, func_details)
+
+                cust_options = config.LAMBDA_DOCKER_OPTIONS
+
                 # Create and start the container
                 LOG.debug('Creating container: %s' % container_name)
                 cmd = (
@@ -230,8 +256,10 @@ def prime_docker_container(self, runtime, func_arn, env_vars, lambda_cwd):
                     ' -e HOSTNAME="$HOSTNAME"'
                     ' -e LOCALSTACK_HOSTNAME="$LOCALSTACK_HOSTNAME"'
                     '  %s'  # env_vars
+                    '  %s'  # network config
+                    '  %s'  # custom options
                     ' lambci/lambda:%s'
-                ) % (container_name, env_vars_str, runtime)
+                ) % (container_name, env_vars_str, network, cust_options, runtime)
                 LOG.debug(cmd)
                 run(cmd, stderr=subprocess.PIPE, outfile=subprocess.PIPE)
 
@@ -403,7 +431,7 @@ def get_container_name(self, func_arn):
 
 class LambdaExecutorSeparateContainers(LambdaExecutorContainers):
 
-    def prepare_execution(self, func_arn, env_vars, runtime, command, handler, lambda_cwd):
+    def prepare_execution(self, func_arn, func_details, env_vars, runtime, command, handler, lambda_cwd):
         entrypoint = ''
         if command:
             entrypoint = ' --entrypoint ""'
@@ -412,25 +440,32 @@ def prepare_execution(self, func_arn, env_vars, runtime, command, handler, lambd
 
         env_vars_string = ' '.join(['-e {}="${}"'.format(k, k) for (k, v) in env_vars.items()])
 
+        network = self.lambda_docker_cmd_networksection(func_arn, func_details)
+        cust_options = config.LAMBDA_DOCKER_OPTIONS
+
         if config.LAMBDA_REMOTE_DOCKER:
             cmd = (
                 'CONTAINER_ID="$(docker create'
                 ' %s'
                 ' %s'
+                ' %s'
+                ' %s'
                 ' "lambci/lambda:%s" %s'
                 ')";'
                 'docker cp "%s/." "$CONTAINER_ID:/var/task";'
                 'docker start -a "$CONTAINER_ID";'
-            ) % (entrypoint, env_vars_string, runtime, command, lambda_cwd)
+            ) % (entrypoint, env_vars_string, network, cust_options, runtime, command, lambda_cwd)
         else:
             lambda_cwd_on_host = self.get_host_path_for_path_in_docker(lambda_cwd)
             cmd = (
                 'docker run'
                 '%s -v "%s":/var/task'
                 ' %s'
+                ' %s'
+                ' %s'
                 ' --rm'
                 ' "lambci/lambda:%s" %s'
-            ) % (entrypoint, lambda_cwd_on_host, env_vars_string, runtime, command)
+            ) % (entrypoint, lambda_cwd_on_host, env_vars_string, network, cust_options, runtime, command)
         return cmd
 
     def get_host_path_for_path_in_docker(self, path):

localstack/services/infra.py

@@ -208,6 +208,7 @@ def setup_logging():
     logging.getLogger('requests').setLevel(logging.WARNING)
     logging.getLogger('botocore').setLevel(logging.ERROR)
     logging.getLogger('elasticsearch').setLevel(logging.ERROR)
+    logging.getLogger('localstack.services.awslambda.lambda_api').setLevel(log_level)
 
 
 def get_service_protocol():

localstack/utils/aws/aws_models.py

@@ -169,6 +169,7 @@ def __init__(self, arn):
         self.handler = None
         self.cwd = None
         self.timeout = None
+        self.vpc_config = None
 
     def get_version(self, version):
         return self.versions.get(version)

tests/integration/lambdas/lambda_python3_nwtest.py

@@ -0,0 +1,13 @@
+# simple test function that identifies the network it is running on
+# note: this requires a web server on the local docker network which will
+# with the alias "networkidentifier" that respond to the endpoingt /network.txt
+#
+# The makefile will set this up before running this test
+
+import requests
+
+
+def handler(event, context):
+    r = requests.get('http://networkidentifier/network.txt')
+    event['network'] = r.text
+    return event

tests/integration/nwfiles/custom/network.txt

@@ -0,0 +1 @@
+custom network

tests/integration/nwfiles/default/network.txt

@@ -0,0 +1 @@
+default network