feat: Add support building layers in DockerInDocker scenario

README.md

@@ -718,6 +718,7 @@ No modules.
 | <a name="input_docker_file"></a> [docker\_file](#input\_docker\_file) | Path to a Dockerfile when building in Docker | `string` | `""` | no |
 | <a name="input_docker_image"></a> [docker\_image](#input\_docker\_image) | Docker image to use for the build | `string` | `""` | no |
 | <a name="input_docker_pip_cache"></a> [docker\_pip\_cache](#input\_docker\_pip\_cache) | Whether to mount a shared pip cache folder into docker environment or not | `any` | `null` | no |
+| <a name="input_docker_volumes_from"></a> [docker\_volumes\_from](#input\_docker\_volumes\_from) | Another container id to mount volumes from (i.e. in DinD scenario) | `string` | `""` | no |
 | <a name="input_docker_with_ssh_agent"></a> [docker\_with\_ssh\_agent](#input\_docker\_with\_ssh\_agent) | Whether to pass SSH\_AUTH\_SOCK into docker environment or not | `bool` | `false` | no |
 | <a name="input_environment_variables"></a> [environment\_variables](#input\_environment\_variables) | A map that defines environment variables for the Lambda Function. | `map(string)` | `{}` | no |
 | <a name="input_ephemeral_storage_size"></a> [ephemeral\_storage\_size](#input\_ephemeral\_storage\_size) | Amount of ephemeral storage (/tmp) in MB your Lambda Function can use at runtime. Valid value between 512 MB to 10,240 MB (10 GB). | `number` | `512` | no |

package.py

@@ -889,6 +889,10 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
         docker_image = docker.docker_image
         docker_build_root = docker.docker_build_root
 
+        if docker_image and not docker_file:
+            docker_cmd = docker_pull_command(image=docker_image)
+            check_call(docker_cmd)
+
         if docker_image:
             ok = False
             while True:
@@ -944,6 +948,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
             if docker:
                 with_ssh_agent = docker.with_ssh_agent
                 pip_cache_dir = docker.docker_pip_cache
+                volumes_from = docker.docker_volumes_from
                 if pip_cache_dir:
                     if isinstance(pip_cache_dir, str):
                         pip_cache_dir = os.path.abspath(
@@ -961,7 +966,7 @@ def install_pip_requirements(query, requirements_file, tmp_dir):
                     '.', shell_command, runtime,
                     image=docker_image_tag_id,
                     shell=True, ssh_agent=with_ssh_agent,
-                    pip_cache_dir=pip_cache_dir,
+                    pip_cache_dir=pip_cache_dir,volumes_from=volumes_from
                 ))
             else:
                 cmd_log.info(shlex_join(pip_command))
@@ -999,6 +1004,10 @@ def install_npm_requirements(query, requirements_file, tmp_dir):
         docker_image = docker.docker_image
         docker_build_root = docker.docker_build_root
 
+        if docker_image and not docker_file:
+            docker_cmd = docker_pull_command(image=docker_image)
+            check_call(docker_cmd)
+
         if docker_image:
             ok = False
             while True:
@@ -1036,6 +1045,7 @@ def install_npm_requirements(query, requirements_file, tmp_dir):
             npm_command = ['npm', 'install']
             if docker:
                 with_ssh_agent = docker.with_ssh_agent
+                volumes_from = docker.docker_volumes_from
                 chown_mask = '{}:{}'.format(os.getuid(), os.getgid())
                 shell_command = [shlex_join(npm_command), '&&',
                                     shlex_join(['chown', '-R',
@@ -1044,7 +1054,7 @@ def install_npm_requirements(query, requirements_file, tmp_dir):
                 check_call(docker_run_command(
                     '.', shell_command, runtime,
                     image=docker_image_tag_id,
-                    shell=True, ssh_agent=with_ssh_agent
+                    shell=True, ssh_agent=with_ssh_agent, volumes_from=volumes_from
                 ))
             else:
                 cmd_log.info(shlex_join(npm_command))
@@ -1061,6 +1071,13 @@ def install_npm_requirements(query, requirements_file, tmp_dir):
             os.remove(target_file)
             yield temp_dir
 
+def docker_pull_command(image):
+    """"""
+    docker_cmd = ['docker', 'pull', image]
+    cmd_log.info(shlex_join(docker_cmd))
+    log_handler and log_handler.flush()
+    return docker_cmd
+
 
 def docker_image_id_command(tag):
     """"""
@@ -1094,20 +1111,24 @@ def docker_build_command(tag=None, docker_file=None, build_root=False):
 
 def docker_run_command(build_root, command, runtime,
                        image=None, shell=None, ssh_agent=False,
-                       interactive=False, pip_cache_dir=None):
+                       interactive=False, pip_cache_dir=None, volumes_from=None):
     """"""
     if platform.system() not in ('Linux', 'Darwin'):
         raise RuntimeError("Unsupported platform for docker building")
 
-    workdir = '/var/task'
+    if volumes_from:
+        workdir = os.path.abspath(build_root)
+    else:
+        workdir = '/var/task'
 
     docker_cmd = ['docker', 'run', '--rm', '-w', workdir]
 
     if interactive:
         docker_cmd.append('-it')
 
-    bind_path = os.path.abspath(build_root)
-    docker_cmd.extend(['-v', "{}:{}:z".format(bind_path, workdir)])
+    if not volumes_from:
+        bind_path = os.path.abspath(build_root)
+        docker_cmd.extend(['-v', "{}:{}:z".format(bind_path, workdir)])
 
     home = os.environ['HOME']
     docker_cmd.extend([
@@ -1143,6 +1164,9 @@ def docker_run_command(build_root, command, runtime,
 
     docker_cmd.extend(['--entrypoint', ''])
 
+    if volumes_from:
+        docker_cmd.extend(['--volumes-from', volumes_from])
+
     docker_cmd.append(image)
 
     assert isinstance(command, list)

package.tf

@@ -17,11 +17,12 @@ data "external" "archive_prepare" {
     })
 
     docker = var.build_in_docker ? jsonencode({
-      docker_pip_cache  = var.docker_pip_cache
-      docker_build_root = var.docker_build_root
-      docker_file       = var.docker_file
-      docker_image      = var.docker_image
-      with_ssh_agent    = var.docker_with_ssh_agent
+      docker_pip_cache    = var.docker_pip_cache
+      docker_build_root   = var.docker_build_root
+      docker_file         = var.docker_file
+      docker_image        = var.docker_image
+      with_ssh_agent      = var.docker_with_ssh_agent
+      docker_volumes_from = var.docker_volumes_from
     }) : null
 
     artifacts_dir = var.artifacts_dir

variables.tf

@@ -671,6 +671,12 @@ variable "docker_pip_cache" {
   default     = null
 }
 
+variable "docker_volumes_from" {
+  description = "Another container id to mount volumes from (i.e. in DinD scenario)"
+  type        = string
+  default     = ""
+}
+
 variable "recreate_missing_package" {
   description = "Whether to recreate missing Lambda package if it is missing locally or not"
   type        = bool