From c72fb76336177f45642f19d82e7d67ecb2dee9fd Mon Sep 17 00:00:00 2001 From: Ian Streeter Date: Wed, 4 May 2022 23:57:36 +0100 Subject: [PATCH] Publish distroless docker image (close #258) --- .github/workflows/lacework.yml | 9 ++++- .github/workflows/test_and_publish.yml | 4 ++ build.sbt | 54 +++++++------------------- project/BuildSettings.scala | 43 ++++++++++++++++---- project/Dependencies.scala | 37 ++++++++++++++++++ project/plugins.sbt | 2 +- 6 files changed, 98 insertions(+), 51 deletions(-) diff --git a/.github/workflows/lacework.yml b/.github/workflows/lacework.yml index ba80c4b..771b73a 100644 --- a/.github/workflows/lacework.yml +++ b/.github/workflows/lacework.yml @@ -29,13 +29,20 @@ jobs: - name: Build docker images run: sbt docker:publishLocal - - name: Scan snowplow-s3-loader + - name: Scan snowplow-s3-loader focal env: LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }} LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }} LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }} run: ./lw-scanner image evaluate snowplow/snowplow-s3-loader ${{ steps.ver.outputs.tag }} --build-id ${{ github.run_id }} --no-pull + - name: Scan snowplow-s3-loader distroless + env: + LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }} + LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }} + LW_SCANNER_SAVE_RESULTS: ${{ !contains(steps.version.outputs.tag, 'rc') }} + run: ./lw-scanner image evaluate snowplow/snowplow-s3-loader ${{ steps.ver.outputs.tag }}-distroless --build-id ${{ github.run_id }} --no-pull + - name: Scan snowplow-s3-loader lzo env: LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }} diff --git a/.github/workflows/test_and_publish.yml b/.github/workflows/test_and_publish.yml index ac167ef..35d6c7c 100644 --- a/.github/workflows/test_and_publish.yml +++ b/.github/workflows/test_and_publish.yml @@ -39,6 +39,10 @@ jobs: if: startsWith(github.ref, 'refs/tags/') run: sbt 'project lzo' docker:publish + - name: Publish to Docker Hub distroless + if: startsWith(github.ref, 'refs/tags/') + run: sbt 'project distroless' docker:publish + - name: Build artifacts run: | sbt assembly diff --git a/build.sbt b/build.sbt index f26f9e3..1750cdd 100644 --- a/build.sbt +++ b/build.sbt @@ -14,56 +14,28 @@ */ lazy val root = project.in(file(".")) - .aggregate(main, lzo) + .aggregate(main, distroless, lzo) lazy val main = project.in(file("modules/main")) + .settings(BuildSettings.mainSettings) .settings( - name := "snowplow-s3-loader", - ) - .settings(BuildSettings.commonSettings) - .settings( - libraryDependencies ++= Seq( - // Java - Dependencies.Libraries.kinesisClient, - Dependencies.Libraries.kinesisConnector, - Dependencies.Libraries.slf4j, - Dependencies.Libraries.jclOverSlf4j, - Dependencies.Libraries.jackson, - Dependencies.Libraries.sentry, - Dependencies.Libraries.jaxbApi, - // Scala - Dependencies.Libraries.decline, - Dependencies.Libraries.circe, - Dependencies.Libraries.snowplowTracker, - Dependencies.Libraries.snowplowBadrows, - Dependencies.Libraries.pureconfig, - Dependencies.Libraries.pureconfigCirce, - // Scala (test only) - Dependencies.Libraries.specs2, - // Thrift (test only) - Dependencies.Libraries.collectorPayload, - Dependencies.Libraries.thrift % Test, - ), - excludeDependencies += "commons-logging" % "commons-logging" + libraryDependencies ++= Dependencies.mainDependencies, + excludeDependencies ++= Dependencies.mainExclusions ) .enablePlugins(JavaAppPackaging, DockerPlugin) -lazy val lzo = project.in(file("modules/lzo")) +lazy val distroless = project.in(file("modules/distroless")) + .settings(BuildSettings.distrolessSettings) + .settings(sourceDirectory := (main / sourceDirectory).value) .settings( - name := "snowplow-s3-loader-lzo", + libraryDependencies ++= Dependencies.mainDependencies, + excludeDependencies ++= Dependencies.mainExclusions ) - .settings(BuildSettings.commonSettings) + .enablePlugins(JavaAppPackaging, DockerPlugin, LauncherJarPlugin) + +lazy val lzo = project.in(file("modules/lzo")) .settings(BuildSettings.lzoSettings) - .settings( - libraryDependencies ++= Seq( - Dependencies.Libraries.hadoop, - Dependencies.Libraries.elephantbird, - Dependencies.Libraries.hadoopLZO, - Dependencies.Libraries.thrift, - Dependencies.Libraries.collections, - Dependencies.Libraries.jacksonCbor, - ) - ) + .settings(libraryDependencies ++= Dependencies.lzoDependencies) .dependsOn(main % "compile->compile; test->test") .enablePlugins(JavaAppPackaging, DockerPlugin) diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 769651d..e53d613 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -15,9 +15,12 @@ import sbt._ import Keys._ -import com.typesafe.sbt.packager.Keys._ -import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport.Docker -import com.typesafe.sbt.packager.docker._ +import com.typesafe.sbt.SbtNativePackager.autoImport._ +import com.typesafe.sbt.packager.archetypes.jar.LauncherJarPlugin.autoImport.packageJavaLauncherJar +import com.typesafe.sbt.packager.docker.{Cmd, DockerPermissionStrategy} +import com.typesafe.sbt.packager.docker.DockerPlugin.autoImport._ +import com.typesafe.sbt.packager.linux.LinuxPlugin.autoImport._ +import com.typesafe.sbt.packager.universal.UniversalPlugin.autoImport._ // Scoverage plugin import scoverage.ScoverageKeys._ @@ -47,7 +50,7 @@ object BuildSettings { } ) - lazy val dockerSettings = Seq( + lazy val dockerSettingsFocal = Seq( Docker / maintainer := "Snowplow Analytics Ltd. ", Docker / daemonUser := "daemon", Docker / packageName := "snowplow/snowplow-s3-loader", @@ -55,13 +58,28 @@ object BuildSettings { dockerUpdateLatest := true, ) - lazy val lzoDockerSettings = Seq( + lazy val dockerSettingsDistroless = Seq( + Docker / maintainer := "Snowplow Analytics Ltd. ", + dockerBaseImage := "gcr.io/distroless/java11-debian11:nonroot", + Docker / daemonUser := "nonroot", + Docker / daemonGroup := "nonroot", + dockerRepository := Some("snowplow"), + Docker / daemonUserUid := None, + Docker / defaultLinuxInstallLocation := "/home/snowplow", + dockerEntrypoint := Seq("java", "-jar",s"/home/snowplow/lib/${(packageJavaLauncherJar / artifactPath).value.getName}"), + dockerPermissionStrategy := DockerPermissionStrategy.CopyChown, + dockerAlias := dockerAlias.value.withTag(Some(version.value + "-distroless")), + dockerUpdateLatest := false + ) + + lazy val lzoDockerSettingsFocal = dockerSettingsFocal ++ Seq( dockerCommands := { val installLzo = Seq(Cmd("RUN", "mkdir -p /var/lib/apt/lists/partial && apt-get update && apt-get install -y lzop && apt-get purge -y")) val (h, t) = dockerCommands.value.splitAt(dockerCommands.value.size-4) h ++ installLzo ++ t }, - dockerAlias := dockerAlias.value.withTag(Some(version.value + "-lzo")) + dockerAlias := dockerAlias.value.withTag(Some(version.value + "-lzo")), + dockerUpdateLatest := false ) // Makes our SBT app settings available from within the app @@ -112,9 +130,18 @@ object BuildSettings { scalafmtOnCompile := false ) - lazy val commonSettings = basicSettings ++ scalifySettings ++ sbtAssemblySettings ++ dockerSettings ++ addExampleConfToTestCp + lazy val commonSettings = basicSettings ++ scalifySettings ++ sbtAssemblySettings ++ addExampleConfToTestCp + + lazy val mainSettings = commonSettings ++ dockerSettingsFocal ++ Seq( + name := "snowplow-s3-loader" + ) + + lazy val distrolessSettings = commonSettings ++ dockerSettingsDistroless ++ Seq( + name := "snowplow-s3-loader" + ) - lazy val lzoSettings = lzoDockerSettings ++ Seq( + lazy val lzoSettings = commonSettings ++ lzoDockerSettingsFocal ++ Seq( + name := "snowplow-s3-loader-lzo", Compile / discoveredMainClasses := Seq(), Compile / mainClass := Some("com.snowplowanalytics.s3.loader.lzo.Main") ) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cb2764e..aa4016f 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -92,4 +92,41 @@ object Dependencies { val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % Test val collectorPayload = "com.snowplowanalytics" % "collector-payload-1" % V.collectorPayload % Test } + + val mainDependencies = Seq( + // Java + Libraries.kinesisClient, + Libraries.kinesisConnector, + Libraries.slf4j, + Libraries.jclOverSlf4j, + Libraries.jackson, + Libraries.sentry, + Libraries.jaxbApi, + // Scala + Libraries.decline, + Libraries.circe, + Libraries.snowplowTracker, + Libraries.snowplowBadrows, + Libraries.pureconfig, + Libraries.pureconfigCirce, + // Scala (test only) + Libraries.specs2, + // Thrift (test only) + Libraries.collectorPayload, + Libraries.thrift % Test + ) + + val lzoDependencies = Seq( + Libraries.hadoop, + Libraries.elephantbird, + Libraries.hadoopLZO, + Libraries.thrift, + Libraries.collections, + Libraries.jacksonCbor, + ) + + val mainExclusions = Seq( + "commons-logging" % "commons-logging" + ) + } diff --git a/project/plugins.sbt b/project/plugins.sbt index 0f9ec2a..ba4703f 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,6 @@ addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.1.18") addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10") -addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.8.1") +addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.7") addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.2") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.0") addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.1")