Separate Docker Layers for Dependencies and App jars. (#1310)

* Docker Layers separate for Dependencies and App jars.

* Use indexed layers to allow flexibility

* avoid mapping `dockerPackageMappings`

* Calculate `dockerLayerMappings` once

It is reused in Docker/stage and and Docker/dockerCommands.
This intermediate step helps with cyclic dependency.

* Dirty version that supports PR code review remarks

* Fix chmod for layers

* Revert changes around Docker / mappings

* Revert changes dockerLayerMappings

* Documenting `dockerLayerGrouping`

* Testing layer groups

* fix tests

* fix more tests

* Version that might pass the tests

* Fix import for sbt 0.1.x

* scalafmt

* Experiment with Option[Int] for layer mapping

* Docker tests of opt-out from layering

* Test of dockerPackageMappings

* Documentation

src/main/scala/com/typesafe/sbt/packager/docker/DockerPlugin.scala

@@ -5,7 +5,7 @@ import java.util.UUID
 import java.util.concurrent.atomic.AtomicBoolean
 
 import sbt._
-import sbt.Keys.{clean, mappings, name, publish, publishLocal, sourceDirectory, streams, target, version}
+import sbt.Keys.{clean, mappings, name, organization, publish, publishLocal, sourceDirectory, streams, target, version}
 import com.typesafe.sbt.packager.Keys._
 import com.typesafe.sbt.packager.linux.LinuxPlugin.autoImport.{daemonUser, defaultLinuxInstallLocation}
 import com.typesafe.sbt.packager.universal.UniversalPlugin
@@ -58,7 +58,7 @@ object DockerPlugin extends AutoPlugin {
   import autoImport._
 
   /**
-    * The separator used by makeCopy should be always forced to UNIX separator.
+    * The separator used by makeCopyLayerIntermediate should be always forced to UNIX separator.
     * The separator doesn't depend on the OS where Dockerfile is being built.
     */
   val UnixSeparatorChar = '/'
@@ -98,6 +98,20 @@ object DockerPlugin extends AutoPlugin {
     ),
     dockerUpdateLatest := false,
     dockerAutoremoveMultiStageIntermediateImages := true,
+    dockerLayerGrouping := {
+      val dockerBaseDirectory = (defaultLinuxInstallLocation in Docker).value
+      (path: String) =>
+        {
+          val pathInWorkdir = path.stripPrefix(dockerBaseDirectory)
+          if (pathInWorkdir.startsWith(s"/lib/${organization.value}"))
+            Some(2)
+          else if (pathInWorkdir.startsWith("/lib/"))
+            Some(1)
+          else if (pathInWorkdir.startsWith("/bin/"))
+            Some(1)
+          else None
+        }
+    },
     dockerAliases := {
       val alias = dockerAlias.value
       if (dockerUpdateLatest.value) {
@@ -123,13 +137,14 @@ object DockerPlugin extends AutoPlugin {
     dockerBuildCommand := dockerExecCommand.value ++ Seq("build") ++ dockerBuildOptions.value ++ Seq("."),
     dockerAdditionalPermissions := {
       val basePath = (defaultLinuxInstallLocation in Docker).value
-      (mappings in Docker).value
+      (dockerLayerMappings in Docker).value
         .collect {
           // by default we assume everything in the bin/ folder should be executable that is not a .bat file
-          case (_, path) if path.startsWith(s"$basePath/bin/") && !path.endsWith(".bat") =>
+          case LayeredMapping(_, _, path) if path.startsWith(s"$basePath/bin/") && !path.endsWith(".bat") =>
             DockerChmodType.UserGroupPlusExecute -> path
           // sh files should also be marked as executable
-          case (_, path) if path.endsWith(".sh") => DockerChmodType.UserGroupPlusExecute -> path
+          case LayeredMapping(_, _, path) if path.endsWith(".sh") =>
+            DockerChmodType.UserGroupPlusExecute -> path
         }
     },
     dockerCommands := {
@@ -142,21 +157,28 @@ object DockerPlugin extends AutoPlugin {
       val base = dockerBaseImage.value
       val addPerms = dockerAdditionalPermissions.value
       val multiStageId = UUID.randomUUID().toString
-
       val generalCommands = makeFrom(base) +: makeMaintainer((maintainer in Docker).value).toSeq
       val stage0name = "stage0"
+      val layerIdsAscending = (dockerLayerMappings in Docker).value.map(_.layerId).distinct.sorted
       val stage0: Seq[CmdLike] = strategy match {
         case DockerPermissionStrategy.MultiStage =>
           Seq(
             makeFromAs(base, stage0name),
             makeLabel("snp-multi-stage" -> "intermediate"),
             makeLabel("snp-multi-stage-id" -> multiStageId),
-            makeWorkdir(dockerBaseDirectory),
-            makeCopy(dockerBaseDirectory),
-            makeUser("root"),
-            makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))
+            makeWorkdir(dockerBaseDirectory)
           ) ++
-            (addPerms map { case (tpe, v) => makeChmod(tpe, Seq(v)) }) ++
+            layerIdsAscending.map(l => makeCopyLayerIntermediate(l, dockerBaseDirectory)) ++
+            Seq(makeUser("root")) ++ layerIdsAscending.map(
+            l => makeChmodRecursive(dockerChmodType.value, Seq(pathInLayer(dockerBaseDirectory, l)))
+          ) ++ {
+            val layerToPath = (dockerLayerGrouping in Docker).value
+            addPerms map {
+              case (tpe, v) =>
+                val layerId = layerToPath(v)
+                makeChmod(tpe, Seq(pathInLayer(v, layerId)))
+            }
+          } ++
             Seq(DockerStageBreak)
         case _ => Seq()
       }
@@ -166,18 +188,22 @@ object DockerPlugin extends AutoPlugin {
           case Some(_) => Seq(makeUser("root"), makeUserAdd(user, group, uidOpt, gidOpt))
           case _       => Seq()
         }) ++
-        Seq(makeWorkdir(dockerBaseDirectory)) ++
+        Seq(makeWorkdir(dockerBaseDirectory)) ++ {
         (strategy match {
           case DockerPermissionStrategy.MultiStage =>
-            Seq(makeCopyFrom(dockerBaseDirectory, stage0name, user, group))
+            layerIdsAscending.map { layerId =>
+              makeCopyFrom(pathInLayer(dockerBaseDirectory, layerId), dockerBaseDirectory, stage0name, user, group)
+            }
           case DockerPermissionStrategy.Run =>
-            Seq(makeCopy(dockerBaseDirectory), makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))) ++
+            layerIdsAscending.map(layerId => makeCopyLayerDirect(layerId, dockerBaseDirectory)) ++
+              Seq(makeChmodRecursive(dockerChmodType.value, Seq(dockerBaseDirectory))) ++
               (addPerms map { case (tpe, v) => makeChmod(tpe, Seq(v)) })
           case DockerPermissionStrategy.CopyChown =>
-            Seq(makeCopyChown(dockerBaseDirectory, user, group))
+            layerIdsAscending.map(layerId => makeCopyChown(layerId, dockerBaseDirectory, user, group))
           case DockerPermissionStrategy.None =>
-            Seq(makeCopy(dockerBaseDirectory))
-        }) ++
+            layerIdsAscending.map(layerId => makeCopyLayerDirect(layerId, dockerBaseDirectory))
+        })
+      } ++
         dockerLabels.value.map(makeLabel) ++
         dockerEnvVars.value.map(makeEnvVar) ++
         makeExposePorts(dockerExposedPorts.value, dockerExposedUdpPorts.value) ++
@@ -231,9 +257,19 @@ object DockerPlugin extends AutoPlugin {
         }
       },
       sourceDirectory := sourceDirectory.value / "docker",
-      stage := Stager.stage(Docker.name)(streams.value, stagingDirectory.value, mappings.value),
+      stage := Stager.stage(Docker.name)(streams.value, stagingDirectory.value, dockerLayerMappings.value.map {
+        case LayeredMapping(layerIdx, file, path) => (file, pathInLayer(path, layerIdx))
+      }),
       stage := (stage dependsOn dockerGenerateConfig).value,
       stagingDirectory := (target in Docker).value / "stage",
+      dockerLayerMappings := {
+        val dockerGroups = dockerLayerGrouping.value
+        val dockerFinalFiles = (mappings in Docker).value
+        for {
+          (file, path) <- dockerFinalFiles
+          layerIdx = dockerGroups(path)
+        } yield LayeredMapping(layerIdx, file, path)
+      },
       target := target.value / "docker",
       // pick a user name that's unlikely to exist in base images
       daemonUser := "demiourgos728",
@@ -316,46 +352,57 @@ object DockerPlugin extends AutoPlugin {
     * @param dockerBaseDirectory the installation directory
     * @return COPY command copying all files inside the installation directory
     */
-  private final def makeCopy(dockerBaseDirectory: String): CmdLike = {
+  private final def makeCopyLayerDirect(layerId: Option[Int], dockerBaseDirectory: String): CmdLike = {
 
     /**
       * This is the file path of the file in the Docker image, and does not depend on the OS where the image
       * is being built. This means that it needs to be the Unix file separator even when the image is built
       * on e.g. Windows systems.
       */
     val files = dockerBaseDirectory.split(UnixSeparatorChar)(1)
-    Cmd("COPY", s"$files /$files")
+    val path = layerId.map(i => s"$i/$files").getOrElse(s"$files")
+    Cmd("COPY", s"$path /$files")
+  }
+
+  private final def makeCopyLayerIntermediate(layerId: Option[Int], dockerBaseDirectory: String): CmdLike = {
+    val files = dockerBaseDirectory.split(UnixSeparatorChar)(1)
+    val path = layerId.map(i => s"$i/$files").getOrElse(s"$files")
+    Cmd("COPY", s"$path /$path")
   }
 
   /**
-    * @param dockerBaseDirectory the installation directory
-    * @param from files are copied from the given build stage
+    * @param src the installation directory
+    * @param stage files are copied from the given build stage
     * @param daemonUser
     * @param daemonGroup
     * @return COPY command copying all files inside the directory from another build stage.
     */
-  private final def makeCopyFrom(dockerBaseDirectory: String,
-                                 from: String,
+  private final def makeCopyFrom(src: String,
+                                 dest: String,
+                                 stage: String,
                                  daemonUser: String,
                                  daemonGroup: String): CmdLike =
-    Cmd("COPY", s"--from=$from --chown=$daemonUser:$daemonGroup $dockerBaseDirectory $dockerBaseDirectory")
+    Cmd("COPY", s"--from=$stage --chown=$daemonUser:$daemonGroup $src $dest")
 
   /**
-    * @param dockerBaseDirectory the installation directory
-    * @param from files are copied from the given build stage
+    * @param layerId the intermediate layer
     * @param daemonUser
     * @param daemonGroup
     * @return COPY command copying all files inside the directory from another build stage.
     */
-  private final def makeCopyChown(dockerBaseDirectory: String, daemonUser: String, daemonGroup: String): CmdLike = {
+  private final def makeCopyChown(layerId: Option[Int],
+                                  dockerBaseDirectory: String,
+                                  daemonUser: String,
+                                  daemonGroup: String): CmdLike = {
 
     /**
       * This is the file path of the file in the Docker image, and does not depend on the OS where the image
       * is being built. This means that it needs to be the Unix file separator even when the image is built
       * on e.g. Windows systems.
       */
     val files = dockerBaseDirectory.split(UnixSeparatorChar)(1)
-    Cmd("COPY", s"--chown=$daemonUser:$daemonGroup $files /$files")
+    val path = layerId.map(i => s"$i/$files").getOrElse(s"$files")
+    Cmd("COPY", s"--chown=$daemonUser:$daemonGroup $path /$files")
   }
 
   /**
@@ -501,6 +548,8 @@ object DockerPlugin extends AutoPlugin {
     inConfig(Docker)(Seq(mappings := renameDests((mappings in Universal).value, defaultLinuxInstallLocation.value)))
   }
 
+  private final def pathInLayer(path: String, layer: Option[Int]) = layer.map(i => s"/$i$path").getOrElse(path)
+
   private[packager] def publishLocalLogger(log: Logger) =
     new sys.process.ProcessLogger {
       override def err(err: => String): Unit =
@@ -716,5 +765,4 @@ object DockerPlugin extends AutoPlugin {
         case _ => List.empty
       }
     }
-
 }

src/main/scala/com/typesafe/sbt/packager/docker/Keys.scala

@@ -57,4 +57,10 @@ private[packager] trait DockerKeysEx extends DockerKeys {
   lazy val dockerAdditionalPermissions =
     taskKey[Seq[(DockerChmodType, String)]]("Explicit chmod calls to some of the paths.")
   val dockerApiVersion = TaskKey[Option[DockerApiVersion]]("dockerApiVersion", "The docker server api version")
+  val dockerLayerGrouping = settingKey[String => Option[Int]](
+    "Group files by path into in layers to increase docker cache hits. " +
+      "Lower index means the file would be a part of an earlier layer."
+  )
+  val dockerLayerMappings =
+    taskKey[Seq[LayeredMapping]]("List of layer, source file and destination in Docker image.")
 }

src/main/scala/com/typesafe/sbt/packager/docker/LayeredMapping.scala

@@ -0,0 +1,15 @@
+package com.typesafe.sbt.packager.docker
+
+import java.io.File
+
+/**
+  * Mapping of file to intermediate layers.
+  *
+  * @param layerId The identifier in the layer used to increase cache hits in
+  *                docker caching. LayerId is present in docker:stage directory structure
+  *                and in intermediate image produced in the multi-stage docker build.
+  *                None means the layering is skipped for this file.
+  * @param file The file produced by universal/stage to be moved into `Docker / stage` directory.
+  * @param path The path in the final image
+  */
+case class LayeredMapping(layerId: Option[Int], file: File, path: String)

src/sbt-test/docker/file-permission/build.sbt

@@ -19,16 +19,19 @@ lazy val root = (project in file("."))
       assert(lines(2).substring(0, 25) == "LABEL snp-multi-stage-id=") // random generated id is hard to test
       assertEquals(lines.drop(3),
         """WORKDIR /opt/docker
-          |COPY opt /opt
+          |COPY 1/opt /1/opt
+          |COPY 2/opt /2/opt
           |USER root
-          |RUN ["chmod", "-R", "u=rX,g=rX", "/opt/docker"]
-          |RUN ["chmod", "u+x,g+x", "/opt/docker/bin/file-permission-test"]
+          |RUN ["chmod", "-R", "u=rX,g=rX", "/1/opt/docker"]
+          |RUN ["chmod", "-R", "u=rX,g=rX", "/2/opt/docker"]
+          |RUN ["chmod", "u+x,g+x", "/1/opt/docker/bin/file-permission-test"]
           |
           |FROM fabric8/java-centos-openjdk8-jdk
           |USER root
           |RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
           |WORKDIR /opt/docker
-          |COPY --from=stage0 --chown=demiourgos728:root /opt/docker /opt/docker
+          |COPY --from=stage0 --chown=demiourgos728:root /1/opt/docker /opt/docker
+          |COPY --from=stage0 --chown=demiourgos728:root /2/opt/docker /opt/docker
           |USER 1001:0
           |ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
           |CMD []""".stripMargin.linesIterator.toList)
@@ -42,7 +45,8 @@ lazy val root = (project in file("."))
           |USER root
           |RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
           |WORKDIR /opt/docker
-          |COPY opt /opt
+          |COPY 1/opt /opt
+          |COPY 2/opt /opt
           |USER 1001:0
           |ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
           |CMD []""".stripMargin.linesIterator.toList)
@@ -56,7 +60,8 @@ lazy val root = (project in file("."))
           |USER root
           |RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 5000 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 5000 sbt || addgroup -g 5000 -S sbt )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 5000 demiourgos728 || adduser -S -u 1001 -G sbt demiourgos728 ))
           |WORKDIR /opt/docker
-          |COPY opt /opt
+          |COPY 1/opt /opt
+          |COPY 2/opt /opt
           |USER 1001:5000
           |ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
           |CMD []""".stripMargin.linesIterator.toList)
@@ -70,7 +75,8 @@ lazy val root = (project in file("."))
           |USER root
           |RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
           |WORKDIR /opt/docker
-          |COPY opt /opt
+          |COPY 1/opt /opt
+          |COPY 2/opt /opt
           |RUN ["chmod", "-R", "u=rX,g=rX", "/opt/docker"]
           |RUN ["chmod", "u+x,g+x", "/opt/docker/bin/file-permission-test"]
           |USER 1001:0
@@ -84,7 +90,8 @@ lazy val root = (project in file("."))
       assertEquals(lines,
         """FROM fabric8/java-centos-openjdk8-jdk
           |WORKDIR /opt/docker
-          |COPY --chown=daemon:root opt /opt
+          |COPY --chown=daemon:root 1/opt /opt
+          |COPY --chown=daemon:root 2/opt /opt
           |USER daemon
           |ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
           |CMD []""".stripMargin.linesIterator.toList)
@@ -99,16 +106,19 @@ lazy val root = (project in file("."))
       assert(lines(2).substring(0, 25) == "LABEL snp-multi-stage-id=") // random generated id is hard to test
       assertEquals(lines.drop(3),
         """WORKDIR /opt/docker
-          |COPY opt /opt
+          |COPY 1/opt /1/opt
+          |COPY 2/opt /2/opt
           |USER root
-          |RUN ["chmod", "-R", "u=rwX,g=rwX", "/opt/docker"]
-          |RUN ["chmod", "u+x,g+x", "/opt/docker/bin/file-permission-test"]
+          |RUN ["chmod", "-R", "u=rwX,g=rwX", "/1/opt/docker"]
+          |RUN ["chmod", "-R", "u=rwX,g=rwX", "/2/opt/docker"]
+          |RUN ["chmod", "u+x,g+x", "/1/opt/docker/bin/file-permission-test"]
           |
           |FROM fabric8/java-centos-openjdk8-jdk
           |USER root
           |RUN id -u demiourgos728 1>/dev/null 2>&1 || (( getent group 0 1>/dev/null 2>&1 || ( type groupadd 1>/dev/null 2>&1 && groupadd -g 0 root || addgroup -g 0 -S root )) && ( type useradd 1>/dev/null 2>&1 && useradd --system --create-home --uid 1001 --gid 0 demiourgos728 || adduser -S -u 1001 -G root demiourgos728 ))
           |WORKDIR /opt/docker
-          |COPY --from=stage0 --chown=demiourgos728:root /opt/docker /opt/docker
+          |COPY --from=stage0 --chown=demiourgos728:root /1/opt/docker /opt/docker
+          |COPY --from=stage0 --chown=demiourgos728:root /2/opt/docker /opt/docker
           |USER 1001:0
           |ENTRYPOINT ["/opt/docker/bin/file-permission-test"]
           |CMD []""".stripMargin.linesIterator.toList)

src/sbt-test/docker/test-executableScriptName/test

@@ -1,6 +1,6 @@
 # Generate the Docker image locally
 > docker:publishLocal
 $ exists target/docker/stage/Dockerfile
-$ exists target/docker/stage/opt/docker/bin/docker-exec
+$ exists target/docker/stage/1/opt/docker/bin/docker-exec
 > checkDockerfile
-$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'
+$ exec bash -c 'docker run docker-package:0.1.0 | grep -q "Hello world"'

src/sbt-test/docker/test-layer-groups/build.sbt

@@ -0,0 +1,10 @@
+enablePlugins(JavaAppPackaging)
+
+organization := "com.example"
+name := "docker-groups"
+version := "0.1.0"
+
+dockerPackageMappings in Docker ++= Seq(
+  (baseDirectory.value / "docker" / "spark-env.sh") -> "/opt/docker/spark/spark-env.sh",
+  (baseDirectory.value / "docker" / "log4j.properties") -> "/opt/docker/spark/log4j.properties"
+)

src/sbt-test/docker/test-layer-groups/changes/nolayers.sbt

@@ -0,0 +1 @@
+dockerLayerGrouping in Docker  := (_ => None)

src/sbt-test/docker/test-layer-groups/docker/log4j.properties

@@ -0,0 +1 @@
+foo=goo

src/sbt-test/docker/test-layer-groups/docker/spark-env.sh

@@ -0,0 +1 @@
+echo "Hello!"

src/sbt-test/docker/test-layer-groups/layers.sbt

@@ -0,0 +1,14 @@
+dockerLayerGrouping in Docker := {
+  val dockerBaseDirectory = (defaultLinuxInstallLocation in Docker).value
+  (path: String) =>
+    {
+      val pathInWorkdir = path.stripPrefix(dockerBaseDirectory)
+      if (pathInWorkdir.startsWith(s"/lib/${organization.value}"))
+        Some(2)
+      else if (pathInWorkdir.startsWith("/bin/"))
+        Some(123)
+      else if (pathInWorkdir.startsWith("/spark/"))
+        Some(54)
+      else None
+    }
+}

src/sbt-test/docker/test-layer-groups/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % sys.props("project.version"))

src/sbt-test/docker/test-layer-groups/src/main/scala/Main.scala

@@ -0,0 +1,3 @@
+object Main extends App {
+  println("Hello world")
+}