Update GH WFs to publish Java images for many platforms

Release and snapshot publishing produces 2 different Docker images: * Java multi-platform / amd64, arm64, ppc64le, s390x (all those are available for OpenJDK 17) * Native / amd64 only The multiplatform Docker images are created via a new shell script using `docker buildx`. This change removes all Docker related settings and dependencies from the Quarkus builds. See notes in `tools/dockerbuild/README.md`
projectnessie · Feb 16, 2023 · 5de59a5 · 5de59a5
1 parent 4f2cfd7
commit 5de59a5
Show file tree

Hide file tree

Showing 11 changed files with 423 additions and 98 deletions.
diff --git a/.github/renovate.json5 b/.github/renovate.json5
@@ -26,6 +26,10 @@
       matchManagers: ["pip_requirements", "pip_setup"],
       "labels": ["dependencies", "python"],
     },
+    {
+      matchManagers: ["docker"],
+      "labels": ["dependencies", "docker"],
+    },
 
     // Check for updates, merge automatically
     {
@@ -58,7 +62,14 @@
         "org.glassfish.jersey:jersey-bom",
       ],
       "matchUpdateTypes": ["minor", "patch", "pin", "digest"],
-    }
+    },
+
+    // Docker
+    {
+      matchManagers: ["docker"],
+      automerge: false,
+      platformAutomerge: false
+    },
   ],
 
   // Max 50 PRs in total, 10 per hour

diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml
@@ -118,22 +118,31 @@ jobs:
         MAVEN_USERNAME: ${{ secrets.OSSRH_ACCESS_ID }}
         MAVEN_OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }}
         MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
+        ARTIFACTS: ../build-artifacts
       run: |
+        rm -rf "${ARTIFACTS}"
+        mkdir -p "${ARTIFACTS}"
+        
         echo "::group::Publish to Sonatype"
         ./gradlew --no-watch-fs --no-daemon publishToMavenLocal publishToSonatype closeAndReleaseSonatypeStagingRepository -Prelease -Puber-jar
+        mv servers/quarkus-server/build/nessie-quarkus-${RELEASE_VERSION}-runner.jar "${ARTIFACTS}"
+        mv servers/quarkus-cli/build/nessie-quarkus-cli-${RELEASE_VERSION}-runner.jar "${ARTIFACTS}"
         echo "::endgroup::"
 
-        # Build the native-image, it's not released to Maven Central
-        echo "::group::Build native image"
-        ./gradlew --no-watch-fs --no-daemon :nessie-quarkus:quarkusBuild -Prelease -Pnative -Pdocker
-        echo "::endgroup::"
+        echo '${{ secrets.DOCKER_TOKEN }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
+        tools/dockerbuild/build-push-images.sh \
+          -a "${ARTIFACTS}" \
+          -g ":nessie-quarkus" \
+          -p "servers/quarkus-server" \
+          -n \
+          projectnessie/nessie
 
         # Add version to the openapi file name
         cp model/build/generated/openapi/META-INF/openapi/openapi.yaml model/build/nessie-openapi-${RELEASE_VERSION}.yaml
 
-        echo "QUARKUS_NATIVE_BINARY=servers/quarkus-server/build/nessie-quarkus-${RELEASE_VERSION}-runner" >> ${GITHUB_ENV}
-        echo "QUARKUS_UBER_JAR=servers/quarkus-server/build/nessie-quarkus-${RELEASE_VERSION}-runner.jar" >> ${GITHUB_ENV}
-        echo "CLI_UBER_JAR=servers/quarkus-cli/build/nessie-quarkus-cli-${RELEASE_VERSION}-runner.jar" >> ${GITHUB_ENV}
+        echo "QUARKUS_UBER_JAR=${ARTIFACTS}/nessie-quarkus-${RELEASE_VERSION}-runner.jar" >> ${GITHUB_ENV}
+        echo "CLI_UBER_JAR=${ARTIFACTS}/nessie-quarkus-cli-${RELEASE_VERSION}-runner.jar" >> ${GITHUB_ENV}
+        echo "QUARKUS_NATIVE_BINARY=${ARTIFACTS}/nessie-quarkus-${RELEASE_VERSION}-runner" >> ${GITHUB_ENV}
         echo "GC_EXEC=gc/gc-shell/build/executable/nessie-gc" >> ${GITHUB_ENV}
         echo "NESSIE_OPENAPI=model/build/nessie-openapi-${RELEASE_VERSION}.yaml" >> ${GITHUB_ENV}
 
@@ -177,26 +186,6 @@ jobs:
     - name: pynessie info
       run: echo "## Successfully published pynessie ${RELEASE_VERSION}" >> $GITHUB_STEP_SUMMARY
 
-    # Deploys Docker images. Build and test steps were already ran in previous workflows
-    - name: Publish Docker image for release
-      run: |
-        echo "## Docker publish ${RELEASE_VERSION}" >> $GITHUB_STEP_SUMMARY
-
-        echo "::group::Tag Docker image"
-        docker tag projectnessie/nessie:${RELEASE_VERSION} projectnessie/nessie:latest
-        echo "Successfully tagged Docker image as \`projectnessie/nessie:${RELEASE_VERSION}\`" >> $GITHUB_STEP_SUMMARY
-        echo "" >> $GITHUB_STEP_SUMMARY
-        echo "::endgroup::"
-
-        echo "::group::Docker Hub login"
-        echo '${{ secrets.DOCKER_TOKEN }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
-        echo "::endgroup::"
-
-        echo "::group::Push Docker image"
-        docker push projectnessie/nessie:${RELEASE_VERSION} && docker push projectnessie/nessie:latest
-        echo "::endgroup::"
-        echo "Successfully pushed Docker image for ${RELEASE_VERSION}" >> $GITHUB_STEP_SUMMARY
-
     # Prepare Nessie release notes for GitHub
     #
     # The markdown file for the release is generated using some mostly simple shell script.
@@ -229,7 +218,10 @@ jobs:
 
         * ${NUM_COMMITS} commits since ${LAST_TAG#nessie-}
         * Maven Central: https://search.maven.org/search?q=g:org.projectnessie+v:${RELEASE_VERSION}
-        * Docker Hub: https://hub.docker.com/r/projectnessie/nessie \`docker pull projectnessie/nessie:${RELEASE_VERSION}\`
+        * Docker Hub: https://hub.docker.com/r/projectnessie/nessie
+          * Multiplatform Java image (amd64 + arm64): `projectnessie/nessie:${RELEASE_VERSION}-java\`
+          * Image with native amd64 binary: `projectnessie/nessie:${RELEASE_VERSION}-native\`
+            (only recommended for short running Nessie instances, amd64 only)
         * PyPI: https://pypi.org/project/pynessie/${RELEASE_VERSION}/
         * Helm Chart repo: https://charts.projectnessie.org/
 

diff --git a/.github/workflows/snapshot-publish.yml b/.github/workflows/snapshot-publish.yml
@@ -38,22 +38,18 @@ jobs:
             publishToSonatype closeAndReleaseSonatypeStagingRepository
             -Prelease -Puber-jar
 
-      - name: Gradle / build Docker container
-        uses: gradle/gradle-build-action@v2
-        with:
-          arguments: |
-            --no-daemon
-            :nessie-quarkus:quarkusBuild
-            -Pnative
-            -Pdocker
-
-      - name: Push Docker images
+      - name: Docker images publishing
         env:
-          DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+          ARTIFACTS: ../build-artifacts
         run: |
+          rm -rf "${ARTIFACTS}"
+          mkdir -p "${ARTIFACTS}"
+
           echo '${{ secrets.DOCKER_TOKEN }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
-          docker images --filter 'reference=projectnessie/nessie' --format '{{.ID}}\t{{.Tag}}' |
-          while read IMAGE_ID IMAGE_TAG; do
-            docker tag "$IMAGE_ID" "projectnessie/nessie-unstable:${IMAGE_TAG%-snapshot}"
-            docker push "projectnessie/nessie-unstable:${IMAGE_TAG%-snapshot}"
-          done
+          tools/dockerbuild/build-push-images.sh \
+            -a "${ARTIFACTS}" \
+            -g ":nessie-quarkus" \
+            -p "servers/quarkus-server" \
+            -n \
+            projectnessie/nessie-unstable
+          rm -rf "${ARTIFACTS}"
diff --git a/servers/lambda/build.gradle.kts b/servers/lambda/build.gradle.kts
@@ -54,19 +54,7 @@ dependencies {
 
 buildForJava11()
 
-val useDocker = project.hasProperty("docker")
 val useNative = project.hasProperty("native")
-var jibPlatforms: String = System.getProperty("quarkus.jib.platforms", "linux/amd64")
-
-if (useNative && jibPlatforms.contains(',')) {
-  val single = jibPlatforms.substring(0, jibPlatforms.indexOf(','))
-  logger.warn(
-    "ONLY building for plaform '{}' instead of '{}', because native image build is enabled.",
-    single,
-    jibPlatforms
-  )
-  jibPlatforms = single
-}
 
 quarkus {
   quarkusBuildProperties.put("quarkus.package.type", quarkusPackageType())
@@ -75,12 +63,6 @@ quarkus {
     libs.versions.quarkusNativeBuilderImage.get()
   )
   quarkusBuildProperties.put("quarkus.native.container-build", useNative.toString())
-  quarkusBuildProperties.put("quarkus.container-image.build", useDocker.toString())
-  quarkusBuildProperties.put(
-    "quarkus.jib.base-jvm-image",
-    libs.versions.quarkusJibBaseJvmImage.get()
-  )
-  quarkusBuildProperties.put("quarkus.jib.platforms", jibPlatforms)
 }
 
 val quarkusBuild by
@@ -98,7 +80,6 @@ tasks.withType<Test>().configureEach {
   systemProperty("quarkus.log.console.level", testLogLevel())
   systemProperty("http.access.log.level", testLogLevel())
   systemProperty("native.image.path", quarkusBuild.nativeRunner)
-  systemProperty("quarkus.container-image.build", useDocker)
   systemProperty("quarkus.smallrye.jwt.enabled", "true")
 
   val testHeapSize: String? by project

diff --git a/servers/quarkus-server/build.gradle.kts b/servers/quarkus-server/build.gradle.kts
@@ -60,7 +60,6 @@ dependencies {
   implementation("io.quarkus:quarkus-micrometer")
   implementation("io.quarkus:quarkus-core-deployment")
   implementation(libs.quarkus.opentelemetry)
-  implementation("io.quarkus:quarkus-container-image-jib")
   implementation(libs.quarkus.logging.sentry)
   implementation("io.smallrye:smallrye-open-api-jaxrs")
   implementation("io.micrometer:micrometer-registry-prometheus")
@@ -114,34 +113,14 @@ val pullOpenApiSpec by
   }
 
 val openApiSpecDir = buildDir.resolve("openapi-extra")
-val useDocker = project.hasProperty("docker")
 val useNative = project.hasProperty("native")
-var jibPlatforms: String = System.getProperty("quarkus.jib.platforms", "linux/amd64")
-
-if (useNative && jibPlatforms.contains(',')) {
-  val single = jibPlatforms.substring(0, jibPlatforms.indexOf(','))
-  logger.warn(
-    "ONLY building for plaform '{}' instead of '{}', because native image build is enabled.",
-    single,
-    jibPlatforms
-  )
-  jibPlatforms = single
-}
 
 quarkus {
   quarkusBuildProperties.put("quarkus.package.type", quarkusPackageType())
   quarkusBuildProperties.put(
     "quarkus.native.builder-image",
     libs.versions.quarkusNativeBuilderImage.get()
   )
-  quarkusBuildProperties.put("quarkus.native.container-build", useNative.toString())
-  quarkusBuildProperties.put("quarkus.container-image.build", useDocker.toString())
-  quarkusBuildProperties.put("quarkus.container-image.builder", "jib")
-  quarkusBuildProperties.put(
-    "quarkus.jib.base-jvm-image",
-    libs.versions.quarkusJibBaseJvmImage.get()
-  )
-  quarkusBuildProperties.put("quarkus.jib.platforms", jibPlatforms)
   quarkusBuildProperties.put(
     "quarkus.smallrye-openapi.store-schema-directory",
     buildDir.resolve("openapi").toString()
@@ -202,7 +181,6 @@ tasks.withType<Test>().configureEach {
   if (project.hasProperty("native")) {
     systemProperty("native.image.path", quarkusBuild.get().nativeRunner)
   }
-  systemProperty("quarkus.container-image.build", useDocker)
   systemProperty("quarkus.smallrye.jwt.enabled", "true")
   systemProperty(
     "it.nessie.container.postgres.tag",

diff --git a/servers/quarkus-server/src/main/resources/application.properties b/servers/quarkus-server/src/main/resources/application.properties
@@ -163,11 +163,6 @@ quarkus.native.additional-build-args=\
   -J-Dfile.encoding=UTF-8,\
   --enable-monitoring
 
-## quarkus container specific settings
-# fixed at buildtime
-quarkus.container-image.group=projectnessie
-quarkus.container-image.name=nessie
-
 # Metrics collection settings
 quarkus.micrometer.enabled=true
 quarkus.micrometer.export.prometheus.enabled=true