From db7b27b9977f29fc660e8db0fd2167b2325d091e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 30 Apr 2025 16:52:37 +0200 Subject: [PATCH 01/66] ci(ct): move CI container scripts to separate directory --- .github/workflows/container_maintenance.yml | 2 +- .github/workflows/scripts/{ => containers}/maintenance-job.sh | 0 .github/workflows/scripts/{ => containers}/utils.sh | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/scripts/{ => containers}/maintenance-job.sh (100%) rename .github/workflows/scripts/{ => containers}/utils.sh (100%) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 986fe25cdf5..7d3e824651b 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -63,7 +63,7 @@ jobs: - name: Execute build matrix script id: execute run: | - .github/workflows/scripts/maintenance-job.sh ${{ steps.discover.outputs.branches }} + .github/workflows/scripts/containers/maintenance-job.sh ${{ steps.discover.outputs.branches }} # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string diff --git a/.github/workflows/scripts/maintenance-job.sh b/.github/workflows/scripts/containers/maintenance-job.sh similarity index 100% rename from .github/workflows/scripts/maintenance-job.sh rename to .github/workflows/scripts/containers/maintenance-job.sh diff --git a/.github/workflows/scripts/utils.sh b/.github/workflows/scripts/containers/utils.sh similarity index 100% rename from .github/workflows/scripts/utils.sh rename to .github/workflows/scripts/containers/utils.sh From b73131fdec402b22a86f437a0151c7c3f17f9ae8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 07:42:12 +0200 Subject: [PATCH 02/66] ci(ct): fix newer parent function repo extraction For the base image we didn't use a parent image which has a namespace. If the given image has a namespace, it needs to be removed from the repo string to create the tags lookup URL. --- .github/workflows/scripts/containers/utils.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/containers/utils.sh b/.github/workflows/scripts/containers/utils.sh index 987b58d8bb5..aac317464ad 100644 --- a/.github/workflows/scripts/containers/utils.sh +++ b/.github/workflows/scripts/containers/utils.sh @@ -9,7 +9,8 @@ function check_newer_parent() { if [[ "$PARENT_IMAGE_NS" = "${PARENT_IMAGE}" ]]; then PARENT_IMAGE_NS="library" fi - PARENT_IMAGE_REPO="${PARENT_IMAGE%:*}" + PARENT_IMAGE_REPO_CUT_NS="${PARENT_IMAGE#*/}" + PARENT_IMAGE_REPO="${PARENT_IMAGE_REPO_CUT_NS%:*}" PARENT_IMAGE_TAG="${PARENT_IMAGE#*:}" PARENT_IMAGE_LAST_UPDATE="$( curl -sS "https://hub.docker.com/v2/namespaces/${PARENT_IMAGE_NS}/repositories/${PARENT_IMAGE_REPO}/tags/${PARENT_IMAGE_TAG}" | jq -r .last_updated )" From cdc8b44b1a6ed1a45f752e78933d6b74681bcc28 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 07:44:01 +0200 Subject: [PATCH 03/66] refactor(ci,ct): move base image maintenance script to other name Done because we want to have a similar script for other images as well. --- .github/workflows/container_maintenance.yml | 2 +- .../scripts/containers/{maintenance-job.sh => maintain-base.sh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/scripts/containers/{maintenance-job.sh => maintain-base.sh} (100%) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 7d3e824651b..531d897a8d4 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -63,7 +63,7 @@ jobs: - name: Execute build matrix script id: execute run: | - .github/workflows/scripts/containers/maintenance-job.sh ${{ steps.discover.outputs.branches }} + .github/workflows/scripts/containers/maintain-base.sh ${{ steps.discover.outputs.branches }} # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string diff --git a/.github/workflows/scripts/containers/maintenance-job.sh b/.github/workflows/scripts/containers/maintain-base.sh similarity index 100% rename from .github/workflows/scripts/containers/maintenance-job.sh rename to .github/workflows/scripts/containers/maintain-base.sh From 6c62a18fb31053ddf1a86fc1a76018fe2851f00b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 07:47:05 +0200 Subject: [PATCH 04/66] doc(ci,ct): add description of base image maintenance job outputs with examples --- .github/workflows/container_maintenance.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 531d897a8d4..28963b2bb6d 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -29,7 +29,12 @@ jobs: # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} outputs: + # This is a JSON map with keys of branch names (supported releases & develop) and values containing an array of known image tags for the branch + # Example: {"v6.6": ["latest", "6.6-noble", "6.6-noble-r1"], "v6.5": ["6.5-noble", "6.5-noble-r5"], "v6.4": ["6.4-noble", "6.4-noble-r12"], "develop": ["unstable", "6.7-noble", "6.7-noble-p6.2025.3-j17"]} supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + + # This is a JSON list containing a flattened map of branch names and the latest non-rolling tag + # Example: [ "v6.6=gdcc/base:6.6-noble-r1", "v6.5=gdcc/base:6.5-noble-r5", "v6.4=gdcc/base:6.4-noble-r12", "develop=gdcc/base:6.7-noble-p6.2025.3-j17" ] rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} steps: From 564977392565aba73774333b9b4accf992d9738f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 07:49:52 +0200 Subject: [PATCH 05/66] refactor(ci,ct): rename base image maintenance job We will add more jobs, so make the job's name more speaking, referring to what happens inside --- .github/workflows/container_maintenance.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 28963b2bb6d..d68d901516d 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -20,7 +20,7 @@ env: NUM_PAST_RELEASES: 3 jobs: - build: + base-image: name: Base Image Matrix Build runs-on: ubuntu-latest permissions: @@ -96,14 +96,14 @@ jobs: permissions: contents: read packages: read - needs: build + needs: base-image steps: - name: Checkout repository uses: actions/checkout@v4 - name: Render README id: render run: | - TAGS_JSON='${{ needs.build.outputs.supported_tag_matrix }}' + TAGS_JSON='${{ needs.base-image.outputs.supported_tag_matrix }}' echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | while IFS= read -r branch; do echo \ From 3636fb11966bffa96e000219589fefb6d49a4756 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 07:57:17 +0200 Subject: [PATCH 06/66] build(ct): add Maven options to skip building app or configbaker image This is necessary so we can have the maintenance jobs only work on their respective parts and not always do both. Otherwise the configbaker image would have many more fixed tags than necessary (the same as the app image has), confusing people because there were no changes. --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 2fed88cb473..a20a23cfbff 100644 --- a/pom.xml +++ b/pom.xml @@ -1073,7 +1073,9 @@ gdcc/dataverse:${app.image.tag} unstable + false false + gdcc/base:${base.image.tag} noble @@ -1081,6 +1083,7 @@ ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} gdcc/configbaker:${conf.image.tag} ${app.image.tag} + false @@ -1122,6 +1125,7 @@ dev_dataverse ${app.image} + ${app.skipBuild} ${docker.platforms} @@ -1151,6 +1155,7 @@ dev_bootstrap ${conf.image} + ${conf.skipBuild} ${docker.platforms} From 3ae2100833be041cdfb8d85d9e58750255573d22 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 08:03:10 +0200 Subject: [PATCH 07/66] build(ct): add an app image version Maven property We will need it to have the version number ready for the maintenance job. At the same time, leaving a note that we will _not_ use it for the default tag we use during development. If we'd use the version there, too, it would break many scripts that now can rely on the single rolling tag of "unstable", as we'd need to change the tags everywhere for a new version. --- modules/dataverse-parent/pom.xml | 2 ++ pom.xml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 87c15709198..7899c74bb44 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -455,6 +455,8 @@ --> ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + + ${base.image.version} diff --git a/pom.xml b/pom.xml index a20a23cfbff..b39951c8ff9 100644 --- a/pom.xml +++ b/pom.xml @@ -1072,6 +1072,10 @@ 17 gdcc/dataverse:${app.image.tag} + unstable false false From c6ee2f5174a9a8bfaed109179e0e8d7bc1590697 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 08:05:01 +0200 Subject: [PATCH 08/66] build(ct): split the base image reference We need this to enable using a different fixed tag when running the maintenance job for the application images. In there, we want to use a fixed tag of a release branch base image. We must enable replacing the default suffix (meant for development purposes) with something else. --- pom.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b39951c8ff9..29c61cc7635 100644 --- a/pom.xml +++ b/pom.xml @@ -1084,7 +1084,10 @@ noble - ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} + ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} + + -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} ${app.image.tag} false From 850bfc000349ffdc355799d5714de87094c9ac31 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 08:15:58 +0200 Subject: [PATCH 09/66] build(ct): add backports of Maven properties for past releases At time of writing, this are 6.4, 6.5 and 6.6 (the latest three releases are supported) --- src/backports/v6.4/001-parent-pom.xml.patch | 10 ++++++ src/backports/v6.4/002-pom.xml.patch | 38 +++++++++++++++++++++ src/backports/v6.5/001-parent-pom.xml.patch | 10 ++++++ src/backports/v6.5/002-pom.xml.patch | 38 +++++++++++++++++++++ src/backports/v6.6/001-parent-pom.xml.patch | 10 ++++++ src/backports/v6.6/002-pom.xml.patch | 38 +++++++++++++++++++++ 6 files changed, 144 insertions(+) create mode 100644 src/backports/v6.4/001-parent-pom.xml.patch create mode 100644 src/backports/v6.4/002-pom.xml.patch create mode 100644 src/backports/v6.5/001-parent-pom.xml.patch create mode 100644 src/backports/v6.5/002-pom.xml.patch create mode 100644 src/backports/v6.6/001-parent-pom.xml.patch create mode 100644 src/backports/v6.6/002-pom.xml.patch diff --git a/src/backports/v6.4/001-parent-pom.xml.patch b/src/backports/v6.4/001-parent-pom.xml.patch new file mode 100644 index 00000000000..b862eae0fe5 --- /dev/null +++ b/src/backports/v6.4/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -448,6 +448,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + gdcc/dataverse:${app.image.tag} + unstable ++ false + false + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} + ${app.image.tag} ++ false + + + +@@ -1046,6 +1050,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1075,6 +1080,7 @@ + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} diff --git a/src/backports/v6.5/001-parent-pom.xml.patch b/src/backports/v6.5/001-parent-pom.xml.patch new file mode 100644 index 00000000000..b862eae0fe5 --- /dev/null +++ b/src/backports/v6.5/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -448,6 +448,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + gdcc/dataverse:${app.image.tag} + unstable ++ false + false + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} + ${app.image.tag} ++ false + + + +@@ -1046,6 +1050,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1075,6 +1080,7 @@ + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} diff --git a/src/backports/v6.6/001-parent-pom.xml.patch b/src/backports/v6.6/001-parent-pom.xml.patch new file mode 100644 index 00000000000..9b2b9dd4b5c --- /dev/null +++ b/src/backports/v6.6/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -455,6 +455,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + gdcc/dataverse:${app.image.tag} + unstable ++ false + false + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} + gdcc/configbaker:${conf.image.tag} + ${app.image.tag} ++ false + + + +@@ -1046,6 +1050,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1075,6 +1080,7 @@ + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} From 2fec56254a6c89dec3ca4cd7bb9158fcc51dfeb8 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 08:31:10 +0200 Subject: [PATCH 10/66] ci(ct): add dry run option to base image maintenance script --- .github/workflows/scripts/containers/maintain-base.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/containers/maintain-base.sh b/.github/workflows/scripts/containers/maintain-base.sh index 370988b9812..e78a488de98 100755 --- a/.github/workflows/scripts/containers/maintain-base.sh +++ b/.github/workflows/scripts/containers/maintain-base.sh @@ -10,6 +10,8 @@ # - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch # - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced # - You added a PLATFORMS env var with all the target platforms you want to build for +# Optional: +# - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out # NOTE: # This script is a culmination of Github Action steps into a single script. @@ -30,6 +32,7 @@ MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" FORCE_BUILD="${FORCE_BUILD:-"0"}" +DRY_RUN="${DRY_RUN:-"0"}" PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" # Setup and validation @@ -130,8 +133,10 @@ for BRANCH in "$@"; do # 8. Let's build the base image if necessary NEWER_IMAGE=0 if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then - mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ - -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + if ! (( DRY_RUN )); then + mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + fi NEWER_IMAGE=1 # Save the information about the immutable or rolling tag we just built if ! (( IS_DEV )); then From 34cf2f5c49eb256176c868dfa631f50d1873543c Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 08:57:17 +0200 Subject: [PATCH 11/66] ci(ct): move branch discovery to separate job We need to reuse the discovered branches in the jobs for the other images, not just the base image. Moving it to a step running first makes the results reusable. --- .github/workflows/container_maintenance.yml | 36 ++++++++++++++------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index d68d901516d..0cae84da02a 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -20,6 +20,25 @@ env: NUM_PAST_RELEASES: 3 jobs: + discover: + name: Discover supported releases + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + outputs: + branches: ${{ steps.discover.outputs.branches }} + develop-branch: ${{ steps.discover.outputs.develop-branch }} + steps: + - name: Discover maintained releases + id: discover + run: | + DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) + echo "develop-branch=$DEVELOPMENT_BRANCH" | tee -a "${GITHUB_OUTPUT}" + + SUPPORTED_BRANCHES=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " ) + echo "branches=$SUPPORTED_BRANCHES" | tee -a "${GITHUB_OUTPUT}" + base-image: name: Base Image Matrix Build runs-on: ubuntu-latest @@ -28,6 +47,8 @@ jobs: packages: read # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} + needs: + - discover outputs: # This is a JSON map with keys of branch names (supported releases & develop) and values containing an array of known image tags for the branch # Example: {"v6.6": ["latest", "6.6-noble", "6.6-noble-r1"], "v6.5": ["6.5-noble", "6.5-noble-r5"], "v6.4": ["6.4-noble", "6.4-noble-r12"], "develop": ["unstable", "6.7-noble", "6.7-noble-p6.2025.3-j17"]} @@ -55,20 +76,13 @@ jobs: with: platforms: ${{ env.PLATFORMS }} - # Discover the releases we want to maintain - - name: Discover maintained releases - id: discover - run: | - echo "FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 )" | tee -a "$GITHUB_ENV" - DEVELOPMENT_BRANCH=$( curl -f -sS https://api.github.com/repos/${{ github.repository }} | jq -r '.default_branch' ) - echo "DEVELOPMENT_BRANCH=$DEVELOPMENT_BRANCH" | tee -a "$GITHUB_ENV" - echo "branches=$( curl -f -sS https://api.github.com/repos/IQSS/dataverse/releases | jq -r " .[0:${{ env.NUM_PAST_RELEASES }}] | .[].tag_name, \"${DEVELOPMENT_BRANCH}\" " | tr "\n" " " )" | tee -a "${GITHUB_OUTPUT}" - # Execute matrix build for the discovered branches - name: Execute build matrix script id: execute - run: | - .github/workflows/scripts/containers/maintain-base.sh ${{ steps.discover.outputs.branches }} + run: > + FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) + DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} + .github/workflows/scripts/containers/maintain-base.sh ${{ needs.discover.outputs.branches }} # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string From be2063db436690d05a6e7cec0335b7ecbf6a4a06 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 08:58:13 +0200 Subject: [PATCH 12/66] ci(ct): temporarily cut the CI jobs loose for testing purposes --- .github/workflows/container_maintenance.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 0cae84da02a..bcf9d4a6599 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -46,7 +46,8 @@ jobs: contents: read packages: read # Only run in upstream repo - avoid unnecessary runs in forks - if: ${{ github.repository_owner == 'IQSS' }} + # TODO: re-enable once we are done testing in gdcc/wip-base-image project + # if: ${{ github.repository_owner == 'IQSS' }} needs: - discover outputs: @@ -79,6 +80,9 @@ jobs: # Execute matrix build for the discovered branches - name: Execute build matrix script id: execute + env: + # TODO: Remove once we are sure this works as intended + DRY_RUN: 1 run: > FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} @@ -111,6 +115,8 @@ jobs: contents: read packages: read needs: base-image + # TODO: Remove once we are sure all of this works... + if: false steps: - name: Checkout repository uses: actions/checkout@v4 From 87be0cb04b4098861204aac3bcd07a933feb8fb3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 09:07:28 +0200 Subject: [PATCH 13/66] ci(ct): add output when base image maintenance script runs in dry-run mode --- .github/workflows/scripts/containers/maintain-base.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/scripts/containers/maintain-base.sh b/.github/workflows/scripts/containers/maintain-base.sh index e78a488de98..53af49c6306 100755 --- a/.github/workflows/scripts/containers/maintain-base.sh +++ b/.github/workflows/scripts/containers/maintain-base.sh @@ -136,6 +136,8 @@ for BRANCH in "$@"; do if ! (( DRY_RUN )); then mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + else + echo "Skipping Maven build as requested by DRY_RUN=1" fi NEWER_IMAGE=1 # Save the information about the immutable or rolling tag we just built From 2f4fbd33b6bee64d41852f9e25f2eee60e50bf81 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 09:08:21 +0200 Subject: [PATCH 14/66] ci(ct): rename output for base-image maintenance job as context is included in job name now --- .github/workflows/container_maintenance.yml | 2 +- .github/workflows/scripts/containers/maintain-base.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index bcf9d4a6599..5feb701e0f5 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -57,7 +57,7 @@ jobs: # This is a JSON list containing a flattened map of branch names and the latest non-rolling tag # Example: [ "v6.6=gdcc/base:6.6-noble-r1", "v6.5=gdcc/base:6.5-noble-r5", "v6.4=gdcc/base:6.4-noble-r12", "develop=gdcc/base:6.7-noble-p6.2025.3-j17" ] - rebuilt_base_images: ${{ steps.execute.outputs.rebuilt_base_images }} + rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} steps: - name: Checkout and Setup Maven diff --git a/.github/workflows/scripts/containers/maintain-base.sh b/.github/workflows/scripts/containers/maintain-base.sh index 53af49c6306..4efdb280e73 100755 --- a/.github/workflows/scripts/containers/maintain-base.sh +++ b/.github/workflows/scripts/containers/maintain-base.sh @@ -174,7 +174,7 @@ for IMAGE in "${REBUILT_BASE_IMAGES[@]}"; do REBUILT_IMAGES+=" \"$IMAGE\" " done REBUILT_IMAGES+="]" -echo "rebuilt_base_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" +echo "rebuilt_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" # Built the supported rolling tags matrix as JSON SUPPORTED_TAGS="{" From 95215eae805048c3489b8021c281d0d495947420 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 09:22:26 +0200 Subject: [PATCH 15/66] ci(ct): add application image maintenance job --- .github/workflows/container_maintenance.yml | 60 ++++-- .../containers/maintain-application.sh | 192 ++++++++++++++++++ 2 files changed, 233 insertions(+), 19 deletions(-) create mode 100755 .github/workflows/scripts/containers/maintain-application.sh diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 5feb701e0f5..bac4afef01e 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -88,25 +88,47 @@ jobs: DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} .github/workflows/scripts/containers/maintain-base.sh ${{ needs.discover.outputs.branches }} - # TODO: Use the needs.build.outputs.rebuilt_base_images with fromJSON() to create a matrix job. - # Must be a single rank matrix (vector), the branch and base image tag information ships as "branch=tag" string - # Will be part of working on #10618, app image versioned tags. - #push-app-img: - # name: "Rebase & Publish App Image" - # permissions: - # contents: read - # packages: write - # pull-requests: write - # secrets: inherit - # needs: - # - build - # strategy: - # fail-fast: false - # matrix: - # branch: ${{ fromJson(needs.discover.outputs.branches) }} - # uses: ./.github/workflows/container_app_push.yml - # with: - # branch: ${{ matrix.branch }} + application-image: + name: "Application Image Matrix Build" + runs-on: ubuntu-latest + needs: + - discover + - base-image + # Only run in upstream repo - avoid unnecessary runs in forks. + # TODO: If we add a push trigger later, we might want to prepend "always() &&" to ignore the status of the base job. + # Needs further investigation. + # TODO: re-enable once we are done testing in gdcc/wip-base-image project + # if: ${{ github.repository_owner == 'IQSS' }} + outputs: + supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + steps: + - name: Checkout and Setup Maven + uses: IQSS/dataverse/.github/actions/setup-maven@develop + with: + pom-paths: ./pom.xml + + # Note: Accessing, pushing tags etc. to DockerHub will only succeed in upstream and + # on events in context of upstream because secrets. PRs run in context of forks by default! + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU for multi-arch builds + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ env.PLATFORMS }} + + # Execute matrix build for the discovered branches + - name: Execute build matrix script + id: execute + env: + # TODO: Remove once we are sure this works as intended + DRY_RUN: 1 + run: > + FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) + DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} + .github/workflows/scripts/containers/maintain-application.sh ${{ needs.discover.outputs.branches }} hub-description: name: Push description to DockerHub diff --git a/.github/workflows/scripts/containers/maintain-application.sh b/.github/workflows/scripts/containers/maintain-application.sh new file mode 100755 index 00000000000..70c8de16565 --- /dev/null +++ b/.github/workflows/scripts/containers/maintain-application.sh @@ -0,0 +1,192 @@ +#!/bin/bash + +# A matrix-like job to maintain a number of releases as well as the latest snap of Dataverse. + +# PREREQUISITES: +# - You have Java, Maven, QEMU and Docker all setup and ready to go +# - You obviously checked out the develop branch, otherwise you'd not be executing this script +# - You added all the branch names you want to run maintenance for as arguments +# Optional, but recommended: +# - You added a DEVELOPMENT_BRANCH env var to your runner/job env with the name of the development branch +# - You added a FORCE_BUILD=0|1 env var to indicate if the base image build should be forced +# - You added a PLATFORMS env var with all the target platforms you want to build for +# Optional: +# - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out + +# NOTE: +# This script is a culmination of Github Action steps into a single script. +# The reason to put all of this in here is due to the complexity of the Github Action and the limitation of the +# matrix support in Github actions, where outputs cannot be aggregated or otherwise used further. + +set -euo pipefail + +# Get all the inputs +# If not within a runner, just print to stdout (duplicating the output in case of tee usage, but that's ok for testing) +GITHUB_OUTPUT=${GITHUB_OUTPUT:-"/proc/self/fd/1"} +GITHUB_ENV=${GITHUB_ENV:-"/proc/self/fd/1"} +GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-"$(pwd)"} +GITHUB_SERVER_URL=${GITHUB_SERVER_URL:-"https://github.com"} +GITHUB_REPOSITORY=${GITHUB_REPOSITORY:-"IQSS/dataverse"} + +MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" + +DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" +FORCE_BUILD="${FORCE_BUILD:-"0"}" +DRY_RUN="${DRY_RUN:-"0"}" +PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" + +# Setup and validation +if [[ -z "$*" ]]; then + >&2 echo "You must give a list of branch names as arguments" + exit 1; +fi + +source "$( dirname "$0" )/utils.sh" + +# Delete old stuff if present +rm -rf "$MAINTENANCE_WORKSPACE" +mkdir -p "$MAINTENANCE_WORKSPACE" + +# Store the image tags we maintain in this array (same order as branches array!) +# This list will be used to build the support matrix within the Docker Hub image description +SUPPORTED_ROLLING_TAGS=() +# Store the tags of application images we are actually rebuilding +# Takes the from "branch-name=app-image-ref" +REBUILT_APP_IMAGES=() + +for BRANCH in "$@"; do + echo "::group::Running maintenance for $BRANCH" + + # 0. Determine if this is a development branch and the most current release + IS_DEV=0 + if [[ "$BRANCH" = "$DEVELOPMENT_BRANCH" ]]; then + IS_DEV=1 + fi + IS_CURRENT_RELEASE=0 + if [[ "$BRANCH" = $( curl -f -sS "https://api.github.com/repos/$GITHUB_REPOSITORY/releases" | jq -r '.[0].tag_name' ) ]]; then + IS_CURRENT_RELEASE=1 + fi + + # 1. Let's get the maintained sources + git clone -c advice.detachedHead=false --depth 1 --branch "$BRANCH" "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" "$MAINTENANCE_WORKSPACE/$BRANCH" + # Switch context + cd "$MAINTENANCE_WORKSPACE/$BRANCH" + + # 2. Now let's apply the patches (we have them checked out in $GITHUB_WORKSPACE, not necessarily in this local checkout) + echo "Checking for patches..." + if [[ -d ${GITHUB_WORKSPACE}/src/backports/$BRANCH ]]; then + echo "Applying patches now." + find "${GITHUB_WORKSPACE}/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + fi + + # 3a. Determine the base image ref (/:) + BASE_IMAGE_REF="" + # For the dev branch we want to full flexi stack tag, to detect stack upgrades requiring new build + if (( IS_DEV )); then + BASE_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=base.image -q -DforceStdout ) + else + # Frist, get the rolling tag of the base image + ROLLING_BASE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + # Now, we want to build any release branch application images on top of a _fixed_ tag, so let's fetch the newest fixed tag + CURRENT_REV=$( current_revision "$ROLLING_BASE_REF" ) + BASE_IMAGE_REF="$ROLLING_BASE_REF-r$CURRENT_REV" + fi + echo "Determined BASE_IMAGE_REF=$BASE_IMAGE_REF from Maven" + + # 3b. Determine the app image ref (/:) + APP_IMAGE_REF="" + if (( IS_DEV )); then + # Results in the rolling tag for the dev branch + APP_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=app.image -q -DforceStdout ) + else + # Results in the rolling tag for the release branch (the fixed tag will be determined from this rolling tag) + # shellcheck disable=SC2016 + APP_IMAGE_REF=$( mvn initialize help:evaluate -Pct -f . -Dexpression=app.image -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout ) + fi + echo "Determined APP_IMAGE_REF=$APP_IMAGE_REF from Maven" + + # 4. Check for Base image updates + NEWER_BASE_IMAGE=0 + if check_newer_parent "$BASE_IMAGE_REF" "$APP_IMAGE_REF"; then + NEWER_BASE_IMAGE=1 + fi + + # 5. Get current immutable revision tag if not on the dev branch + REV=$( current_revision "$APP_IMAGE_REF" ) + CURRENT_REV_TAG="${APP_IMAGE_REF#*:}-r$REV" + NEXT_REV_TAG="${APP_IMAGE_REF#*:}-r$(( REV + 1 ))" + + # 6. Let's put together what tags we want added to this build run + TAG_OPTIONS="" + if ! (( IS_DEV )); then + TAG_OPTIONS="-Dapp.image=$APP_IMAGE_REF -Ddocker.tags.revision=$NEXT_REV_TAG" + # In case of the current release, add the "latest" tag as well. + if (( IS_CURRENT_RELEASE )); then + TAG_OPTIONS="$TAG_OPTIONS -Ddocker.tags.latest=latest" + fi + else + # shellcheck disable=SC2016 + UPCOMING_TAG=$( mvn initialize help:evaluate -Pct -f . -Dexpression=app.image.tag -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout ) + TAG_OPTIONS="-Ddocker.tags.upcoming=$UPCOMING_TAG" + + # For the dev branch we only have rolling tags and can add them now already + SUPPORTED_ROLLING_TAGS+=("[\"unstable\", \"$UPCOMING_TAG\"]") + fi + echo "Determined these additional Maven tag options: $TAG_OPTIONS" + + # 8. Let's build the base image if necessary + NEWER_IMAGE=0 + if (( NEWER_BASE_IMAGE + FORCE_BUILD > 0 )); then + if ! (( DRY_RUN )); then + # Build the application image, but skip the configbaker image (that's a different job)! + mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ + -Dconf.skipBuild -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + else + echo "Skipping Maven build as requested by DRY_RUN=1" + fi + NEWER_IMAGE=1 + # Save the information about the immutable or rolling tag we just built + if ! (( IS_DEV )); then + REBUILT_APP_IMAGES+=("$BRANCH=${APP_IMAGE_REF%:*}:$NEXT_REV_TAG") + else + REBUILT_APP_IMAGES+=("$BRANCH=$APP_IMAGE_REF") + fi + else + echo "No rebuild necessary, we're done here." + fi + + # 9. Add list of rolling and immutable tags for release builds + if ! (( IS_DEV )); then + RELEASE_TAGS_LIST="[" + if (( IS_CURRENT_RELEASE )); then + RELEASE_TAGS_LIST+="\"latest\", " + fi + RELEASE_TAGS_LIST+="\"${APP_IMAGE_REF#*:}\", " + if (( NEWER_IMAGE )); then + RELEASE_TAGS_LIST+="\"$NEXT_REV_TAG\"]" + else + RELEASE_TAGS_LIST+="\"$CURRENT_REV_TAG\"]" + fi + SUPPORTED_ROLLING_TAGS+=("${RELEASE_TAGS_LIST}") + fi + + echo "::endgroup::" +done + +# Built the output which images have actually been rebuilt as JSON +REBUILT_IMAGES="[" +for IMAGE in "${REBUILT_APP_IMAGES[@]}"; do + REBUILT_IMAGES+=" \"$IMAGE\" " +done +REBUILT_IMAGES+="]" +echo "rebuilt_images=${REBUILT_IMAGES// /, }" | tee -a "${GITHUB_OUTPUT}" + +# Built the supported rolling tags matrix as JSON +SUPPORTED_TAGS="{" +for (( i=0; i < ${#SUPPORTED_ROLLING_TAGS[@]} ; i++ )); do + j=$((i+1)) + SUPPORTED_TAGS+="\"${!j}\": ${SUPPORTED_ROLLING_TAGS[$i]}" + (( i < ${#SUPPORTED_ROLLING_TAGS[@]}-1 )) && SUPPORTED_TAGS+=", " +done +SUPPORTED_TAGS+="}" +echo "supported_tag_matrix=$SUPPORTED_TAGS" | tee -a "$GITHUB_OUTPUT" From b27b4b17bd6071feab46c0a181810b72a9c7b21f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 10:06:29 +0200 Subject: [PATCH 16/66] ci(ct): add damp-run mode to maintenance scripts With damp run we actually build the image, but refrain from the deployment step. This way one may check if the Maven args are all what they should look like. --- .../scripts/containers/maintain-application.sh | 9 +++++++++ .github/workflows/scripts/containers/maintain-base.sh | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/containers/maintain-application.sh b/.github/workflows/scripts/containers/maintain-application.sh index 70c8de16565..2ced82b6b75 100755 --- a/.github/workflows/scripts/containers/maintain-application.sh +++ b/.github/workflows/scripts/containers/maintain-application.sh @@ -12,6 +12,7 @@ # - You added a PLATFORMS env var with all the target platforms you want to build for # Optional: # - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out +# - Use DAMP_RUN=1 env var to skip pushing images, but build them # NOTE: # This script is a culmination of Github Action steps into a single script. @@ -33,6 +34,7 @@ MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" FORCE_BUILD="${FORCE_BUILD:-"0"}" DRY_RUN="${DRY_RUN:-"0"}" +DAMP_RUN="${DAMP_RUN:-"0"}" PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" # Setup and validation @@ -41,6 +43,11 @@ if [[ -z "$*" ]]; then exit 1; fi +if (( DRY_RUN + DAMP_RUN > 1 )); then + >&2 echo "You must either use DRY_RUN=1 or DAMP_RUN=1, but not both" + exit 1; +fi + source "$( dirname "$0" )/utils.sh" # Delete old stuff if present @@ -139,8 +146,10 @@ for BRANCH in "$@"; do if (( NEWER_BASE_IMAGE + FORCE_BUILD > 0 )); then if ! (( DRY_RUN )); then # Build the application image, but skip the configbaker image (that's a different job)! + # shellcheck disable=SC2046 mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ -Dconf.skipBuild -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) else echo "Skipping Maven build as requested by DRY_RUN=1" fi diff --git a/.github/workflows/scripts/containers/maintain-base.sh b/.github/workflows/scripts/containers/maintain-base.sh index 4efdb280e73..8ff5334fd2e 100755 --- a/.github/workflows/scripts/containers/maintain-base.sh +++ b/.github/workflows/scripts/containers/maintain-base.sh @@ -12,6 +12,7 @@ # - You added a PLATFORMS env var with all the target platforms you want to build for # Optional: # - Use DRY_RUN=1 env var to skip actually building, but see how the tag lookups play out +# - Use DAMP_RUN=1 env var to skip pushing images, but build them # NOTE: # This script is a culmination of Github Action steps into a single script. @@ -33,6 +34,7 @@ MAINTENANCE_WORKSPACE="${GITHUB_WORKSPACE}/maintenance-job" DEVELOPMENT_BRANCH="${DEVELOPMENT_BRANCH:-"develop"}" FORCE_BUILD="${FORCE_BUILD:-"0"}" DRY_RUN="${DRY_RUN:-"0"}" +DAMP_RUN="${DAMP_RUN:-"0"}" PLATFORMS="${PLATFORMS:-"linux/amd64,linux/arm64"}" # Setup and validation @@ -41,6 +43,11 @@ if [[ -z "$*" ]]; then exit 1; fi +if (( DRY_RUN + DAMP_RUN > 1 )); then + >&2 echo "You must either use DRY_RUN=1 or DAMP_RUN=1, but not both" + exit 1; +fi + source "$( dirname "$0" )/utils.sh" # Delete old stuff if present @@ -134,8 +141,10 @@ for BRANCH in "$@"; do NEWER_IMAGE=0 if (( NEWER_JAVA_IMAGE + NEWER_PKGS + FORCE_BUILD > 0 )); then if ! (( DRY_RUN )); then + # shellcheck disable=SC2046 mvn -Pct -f modules/container-base deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ - -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ + $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) else echo "Skipping Maven build as requested by DRY_RUN=1" fi From 5918bd3c4f2714b9c3e08525fdeb0e0cf01bdb1e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 10:08:05 +0200 Subject: [PATCH 17/66] ci(ct): fix app image maintenance script missing base image tag We determine the fixed base image tag before, but it wasn't handed to the build as a property. --- .github/workflows/scripts/containers/maintain-application.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scripts/containers/maintain-application.sh b/.github/workflows/scripts/containers/maintain-application.sh index 2ced82b6b75..9e72457a945 100755 --- a/.github/workflows/scripts/containers/maintain-application.sh +++ b/.github/workflows/scripts/containers/maintain-application.sh @@ -148,7 +148,8 @@ for BRANCH in "$@"; do # Build the application image, but skip the configbaker image (that's a different job)! # shellcheck disable=SC2046 mvn -Pct -f . deploy -Ddocker.noCache -Ddocker.platforms="${PLATFORMS}" \ - -Dconf.skipBuild -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS + -Dconf.skipBuild -Dbase.image="${BASE_IMAGE_REF}" \ + -Ddocker.imagePropertyConfiguration=override $TAG_OPTIONS \ $( if (( DAMP_RUN )); then echo "-Ddocker.skip.push -Ddocker.skip.tag"; fi ) else echo "Skipping Maven build as requested by DRY_RUN=1" From c6001c4796019931b745ea11ade1a33194cc552b Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 10:18:12 +0200 Subject: [PATCH 18/66] ci(ct): add input options for dry and damp runs in maintenance workflow --- .github/workflows/container_maintenance.yml | 23 ++++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index bac4afef01e..44060aefd27 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -12,6 +12,16 @@ on: required: false default: false description: "Build and deploy even if no newer Java images or package updates are found." + dry_run: + type: boolean + required: false + default: false + description: "Run in dry-run mode (no builds, verify logic)" + damp_run: + type: boolean + required: false + default: false + description: "Run in damp-run mode (build but don't push)" schedule: - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC @@ -80,11 +90,10 @@ jobs: # Execute matrix build for the discovered branches - name: Execute build matrix script id: execute - env: - # TODO: Remove once we are sure this works as intended - DRY_RUN: 1 run: > FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) + DRY_RUN=$( [[ "${{ inputs.dry_run }}" = "true" ]] && echo 1 || echo 0 ) + DAMP_RUN=$( [[ "${{ inputs.damp_run }}" = "true" ]] && echo 1 || echo 0 ) DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} .github/workflows/scripts/containers/maintain-base.sh ${{ needs.discover.outputs.branches }} @@ -122,11 +131,10 @@ jobs: # Execute matrix build for the discovered branches - name: Execute build matrix script id: execute - env: - # TODO: Remove once we are sure this works as intended - DRY_RUN: 1 run: > FORCE_BUILD=$( [[ "${{ inputs.force_build }}" = "true" ]] && echo 1 || echo 0 ) + DRY_RUN=$( [[ "${{ inputs.dry_run }}" = "true" ]] && echo 1 || echo 0 ) + DAMP_RUN=$( [[ "${{ inputs.damp_run }}" = "true" ]] && echo 1 || echo 0 ) DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} .github/workflows/scripts/containers/maintain-application.sh ${{ needs.discover.outputs.branches }} @@ -137,8 +145,6 @@ jobs: contents: read packages: read needs: base-image - # TODO: Remove once we are sure all of this works... - if: false steps: - name: Checkout repository uses: actions/checkout@v4 @@ -157,6 +163,7 @@ jobs: sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" - name: Push description to DockerHub + if: ${{ ! inputs.dry_run && ! inputs.damp_run }} uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} From f1da695e984eedc8b3819679188b96e724f9675f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 10:18:49 +0200 Subject: [PATCH 19/66] ci(ct): print the container readme to the logs for debugging purposes within maintenance workflow --- .github/workflows/container_maintenance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 44060aefd27..cb4e258f6f8 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -161,6 +161,7 @@ jobs: | tee -a "${GITHUB_WORKSPACE}/tags.md" done sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" + cat "./modules/container-base/README.md" - name: Push description to DockerHub if: ${{ ! inputs.dry_run && ! inputs.damp_run }} From fc4b2a7964bdb74af364fc21196cf66b5fb23cce Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 11:23:59 +0200 Subject: [PATCH 20/66] ci(ct): fix maintenance scripts to make the patch command ignore whitespace problems --- .github/workflows/scripts/containers/maintain-application.sh | 2 +- .github/workflows/scripts/containers/maintain-base.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scripts/containers/maintain-application.sh b/.github/workflows/scripts/containers/maintain-application.sh index 9e72457a945..b68c2a53d96 100755 --- a/.github/workflows/scripts/containers/maintain-application.sh +++ b/.github/workflows/scripts/containers/maintain-application.sh @@ -83,7 +83,7 @@ for BRANCH in "$@"; do echo "Checking for patches..." if [[ -d ${GITHUB_WORKSPACE}/src/backports/$BRANCH ]]; then echo "Applying patches now." - find "${GITHUB_WORKSPACE}/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + find "${GITHUB_WORKSPACE}/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -l -s -i fi # 3a. Determine the base image ref (/:) diff --git a/.github/workflows/scripts/containers/maintain-base.sh b/.github/workflows/scripts/containers/maintain-base.sh index 8ff5334fd2e..5b9ae738b98 100755 --- a/.github/workflows/scripts/containers/maintain-base.sh +++ b/.github/workflows/scripts/containers/maintain-base.sh @@ -83,7 +83,7 @@ for BRANCH in "$@"; do echo "Checking for patches..." if [[ -d ${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH ]]; then echo "Applying patches now." - find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -s -i + find "${GITHUB_WORKSPACE}/modules/container-base/src/backports/$BRANCH" -type f -name '*.patch' -print0 | xargs -0 -n1 patch -p1 -l -s -i fi # 3. Determine the base image ref (/:) From 30bbcd22bcd71487241072edc401c1d5ae8d41ee Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 11:56:12 +0200 Subject: [PATCH 21/66] ci: no longer run Maven test for master or tags It doesn't make sense to run unit tests for released tags (that's too late anyway). It doesn't make sense to run unit tests _again_ when there is a merge to master - this was done on the PR for master already. The most important side effect: don't call the app image release process from here, leave this to the maintenance job. --- .github/workflows/maven_unit_test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 45180ea7aec..44c6117dd9c 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -2,6 +2,11 @@ name: Maven Tests on: push: + # Only run for development and feature branches but not tags or the stable branch. + # For tags (=released) it would be too late anyway, and there is always a PR for the master branch. + branches: + - '*' + - '!master' paths: - "**.java" - "**.sql" From 137a06b42b24d61e5b3105913ae95361ded64764 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 12:08:48 +0200 Subject: [PATCH 22/66] ci(ct): run the maintenance workflow for new releases, too This way we keep workflows for development and releases (and the maintenance of releases) separate. --- .github/workflows/container_maintenance.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index cb4e258f6f8..b77d8b78a2d 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -24,6 +24,8 @@ on: description: "Run in damp-run mode (build but don't push)" schedule: - cron: '23 3 * * 0' # Run for 'develop' every Sunday at 03:23 UTC + release: + types: [published] env: PLATFORMS: linux/amd64,linux/arm64 From a638dbac86d0ebfd6a682dc1ba6c3f4fc3878d78 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 12:20:30 +0200 Subject: [PATCH 23/66] ci(ct): simplify the base image push workflow - No longer react to tags. Leave the release stuff to the maintenance job. - Makes it much easier to determine names and stuff --- .github/workflows/container_base_push.yml | 41 ++++++----------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/.github/workflows/container_base_push.yml b/.github/workflows/container_base_push.yml index 3830fd2f99f..3b375e13864 100644 --- a/.github/workflows/container_base_push.yml +++ b/.github/workflows/container_base_push.yml @@ -1,10 +1,8 @@ --- -name: Container Images Releasing +name: Base Container Image on: push: - tags: - - 'v[6-9].**' branches: - 'develop' # "Path filters are not evaluated for pushes of tags" https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#onpushpull_requestpull_request_targetpathspaths-ignore @@ -33,7 +31,7 @@ jobs: # Only run in upstream repo - avoid unnecessary runs in forks if: ${{ github.repository_owner == 'IQSS' }} outputs: - base-image-ref: ${{ steps.finalize.outputs.base-image-ref }} + base-image-ref: ${{ steps.determine-name.outputs.full-ref }} steps: - name: Checkout and Setup Maven @@ -52,12 +50,10 @@ jobs: # In case this is a push to develop, we care about buildtime. # Configure a remote ARM64 build host in addition to the local AMD64 in two steps. - name: Setup SSH agent - if: ${{ github.event_name != 'schedule' }} uses: webfactory/ssh-agent@v0.9.1 with: ssh-private-key: ${{ secrets.BUILDER_ARM64_SSH_PRIVATE_KEY }} - name: Provide the known hosts key and the builder config - if: ${{ github.event_name != 'schedule' }} run: | echo "${{ secrets.BUILDER_ARM64_SSH_HOST_KEY }}" > ~/.ssh/known_hosts mkdir -p modules/container-base/target/buildx-state/buildx/instances @@ -81,23 +77,17 @@ jobs: # Determine the base image name we are going to use from here on - name: Determine base image name + id: determine-name run: | - if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - echo "BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - else - echo "BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout )" | tee -a "${GITHUB_ENV}" - fi - - name: Calculate revision number for immutable tag (on release branches only) - if: ${{ github.ref_name != env.DEVELOPMENT_BRANCH }} - id: revision-tag - uses: ./.github/actions/get-image-revision - with: - image-ref: ${{ env.BASE_IMAGE }} - tag-options-prefix: "-Dbase.image.tag.suffix='' -Ddocker.tags.revision=" + BASE_IMAGE=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -q -DforceStdout ) + BASE_IMAGE_UPCOMING=$( mvn initialize help:evaluate -Pct -f modules/container-base -Dexpression=base.image -Dbase.image.tag.suffix="" -q -DforceStdout ) + + echo "BASE_IMAGE=${BASE_IMAGE}" | tee -a "${GITHUB_ENV}" + echo "BASE_IMAGE_UPCOMING=${BASE_IMAGE_UPCOMING}" | tee -a "${GITHUB_ENV}" + echo "full-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" + - name: Configure update of "latest" tag for development branch id: develop-tag - if: ${{ github.ref_name == env.DEVELOPMENT_BRANCH }} run: | echo "tag-options=-Ddocker.tags.develop=unstable -Ddocker.tags.upcoming=${BASE_IMAGE_UPCOMING#*:}" | tee -a "${GITHUB_OUTPUT}" @@ -105,16 +95,7 @@ jobs: id: build run: | mvn -f modules/container-base -Pct deploy -Ddocker.noCache -Ddocker.platforms=${{ env.PLATFORMS }} \ - -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} ${{ steps.revision-tag.outputs.tag-options }} - - - name: Determine appropriate base image ref for app image - id: finalize - run: | - if [[ "${{ github.ref_name }}" = "${{ env.DEVELOPMENT_BRANCH }}" ]]; then - echo "base-image-ref=${BASE_IMAGE_UPCOMING}" | tee -a "$GITHUB_OUTPUT" - else - echo "base-image-ref=gdcc/base:${{ steps.revision-tag.outputs.revision-tag }}" | tee -a "$GITHUB_OUTPUT" - fi + -Ddocker.imagePropertyConfiguration=override ${{ steps.develop-tag.outputs.tag-options }} push-app-img: name: "Rebase & Publish App Image" From 26af68654b8339c66ef9e9e52b9507ba382c9f44 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 12:22:37 +0200 Subject: [PATCH 24/66] ci(ct): simplify the app image push workflow - Remove handling the Hub descriptions (leave it to the maintenance workflow) - Remove handling the handling of the master branch (leave it to the maintenance workflow) --- .github/workflows/container_app_push.yml | 43 +++++------------------- 1 file changed, 9 insertions(+), 34 deletions(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 71ffffb5f48..44a67ce86fa 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -60,30 +60,6 @@ jobs: # TODO: add smoke / integration testing here (add "-Pct -DskipIntegrationTests=false") - hub-description: - needs: build - name: Push image descriptions to Docker Hub - # Run this when triggered via push or schedule as reused workflow from base / maven unit tests. - # Excluding PRs here means we will have no trouble with secrets access. Also avoid runs in forks. - if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/dataverse - short-description: "Dataverse Application Container Image providing the executable" - readme-filepath: ./src/main/docker/README.md - - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: gdcc/configbaker - short-description: "Dataverse Config Baker Container Image providing setup tooling and more" - readme-filepath: ./modules/container-configbaker/README.md - # Note: Accessing, pushing tags etc. to DockerHub or GHCR will only succeed in upstream because secrets. # We check for them here and subsequent jobs can rely on this to decide if they shall run. check-secrets: @@ -107,13 +83,13 @@ jobs: needs: check-secrets name: "Package & Publish" runs-on: ubuntu-latest - # Only run this job if we have access to secrets. This is true for events like push/schedule which run in + # Only run this job if we have access to secrets. This is true for events like push/schedule which run in the # context of the main repo, but for PRs only true if coming from the main repo! Forks have no secret access. # # Note: The team's decision was to not auto-deploy an image on any git push where no PR exists (yet). - # Accordingly, only run for push events on branches develop and master. + # Accordingly, only run for push events on the 'develop' branch. if: needs.check-secrets.outputs.available == 'true' && - ( github.event_name != 'push' || ( github.event_name == 'push' && contains(fromJSON('["develop", "master"]'), github.ref_name))) + ( github.event_name != 'push' || ( github.event_name == 'push' && github.ref_name == 'develop' )) steps: - name: Checkout and Setup Maven uses: IQSS/dataverse/.github/actions/setup-maven@develop @@ -141,16 +117,15 @@ jobs: - name: Set up QEMU for multi-arch builds uses: docker/setup-qemu-action@v3 - - name: Re-set image tag based on branch (if master) - if: ${{ github.ref_name == 'master' }} + - name: Add rolling image tag when pushing to develop + if: ${{ github.event_name == 'push' && github.ref_name == 'develop' }} run: | - echo "IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV - echo "BASE_IMAGE_TAG=${{ env.MASTER_BRANCH_TAG }}" >> $GITHUB_ENV + echo "ADDITIONAL_TAGS=-Ddocker.tags.upcoming=$( mvn initialize help:evaluate -Pct -Dexpression=app.image.tag -Dapp.image.tag='${app.image.version}-${base.image.flavor}' -q -DforceStdout )" | tee -a "$GITHUB_ENV" - name: Re-set image tag and container registry when on PR if: ${{ github.event_name == 'pull_request' }} run: | - echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" >> $GITHUB_ENV - echo "REGISTRY='-Ddocker.registry=ghcr.io'" >> $GITHUB_ENV + echo "IMAGE_TAG=$(echo "$GITHUB_HEAD_REF" | tr '\\/_:&+,;#*' '-')" | tee -a "$GITHUB_ENV" + echo "REGISTRY='-Ddocker.registry=ghcr.io'" | tee -a "$GITHUB_ENV" # Necessary to split as otherwise the submodules are not available (deploy skips install) - name: Build app and configbaker container image with local architecture and submodules (profile will skip tests) @@ -162,7 +137,7 @@ jobs: - name: Deploy multi-arch application and configbaker container image run: > mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} + -Dapp.image.tag=${{ env.IMAGE_TAG }} ${{ env.ADDITIONAL_TAGS }} $( [[ -n "${{ inputs.base-image-ref }}" ]] && echo "-Dbase.image=${{ inputs.base-image-ref }}" ) ${{ env.REGISTRY }} -Ddocker.platforms=${{ env.PLATFORMS }} -P ct deploy From 7da0361ccf15fdafd985ad4f8697c10f8411a53e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 12:24:57 +0200 Subject: [PATCH 25/66] ci(ct): fix app image workflow for external PRs - Do not pin down the base image name, we want Maven to take care of that (so we use the long rolling tag for the unstable branch) - Leave a comment that the workflow is simple enough to be included with the main app push flow. This is due to the fact that the app push flow no longer cares about releases, just dev time things --- .github/workflows/container_app_pr.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index 4a06cb567b0..a4c52805156 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -1,6 +1,8 @@ --- name: Preview Application Container Image +# TODO: merge this workflow into the existing container_app_push.yaml flow - there's not much difference! + on: # We only run the push commands if we are asked to by an issue comment with the correct command. # This workflow is always taken from the default branch and runs in repo context with access to secrets. @@ -8,8 +10,6 @@ on: types: [ push-image-command ] env: - IMAGE_TAG: unstable - BASE_IMAGE_TAG: unstable PLATFORMS: "linux/amd64,linux/arm64" jobs: @@ -65,10 +65,9 @@ jobs: install - name: Deploy multi-arch application and configbaker container image run: > - mvn - -Dapp.image.tag=${{ env.IMAGE_TAG }} -Dbase.image.tag=${{ env.BASE_IMAGE_TAG }} + mvn -Pct deploy + -Dapp.image.tag=${{ env.IMAGE_TAG }} -Ddocker.registry=ghcr.io -Ddocker.platforms=${{ env.PLATFORMS }} - -Pct deploy - uses: marocchino/sticky-pull-request-comment@v2 with: From bcf9a9df907ccea3e02f030f6059d4382bd0465f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 12:44:18 +0200 Subject: [PATCH 26/66] doc(ct): adapt app image README for Hub to new tag policy --- src/main/docker/README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/docker/README.md b/src/main/docker/README.md index 06e2769ed6e..48416c196ca 100644 --- a/src/main/docker/README.md +++ b/src/main/docker/README.md @@ -30,13 +30,18 @@ to ask for help and guidance. ## Supported Image Tags This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). -Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/src/main/docker) -happens there (again, by the community). Community-supported image tags are based on the two most important branches: +Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/src/main/docker) happens there (again, by the community). -- The `unstable` tag corresponds to the `develop` branch, where pull requests are merged. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/src/main/docker/Dockerfile)) -- The `alpha` tag corresponds to the `master` branch, where releases are cut from. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/src/main/docker/Dockerfile)) +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). +For more detailed information about our tagging policy, please read about our [application image tags](https://guides.dataverse.org/en/latest/container/app-image.html#supported-image-tags) in the Dataverse Containers Guide. + +For ease of use, here is a list of images that are currently maintained. + + + +All of them are rolling tags, except those ending with `-r`, which are the most recent immutable tags. +The `unstable` tags are the current development branch snapshot. +We strongly recommend using only immutable tags for production use cases. Within the main repository, you may find the application image files at `/src/main/docker`. This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. From 6c3e60b2f1cbcb59356e6960948454bd973e59ad Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 12:45:07 +0200 Subject: [PATCH 27/66] ci(ct): add hub description deployment steps to maintenance workflow --- .github/workflows/container_maintenance.yml | 75 ++++++++++++++++++--- 1 file changed, 66 insertions(+), 9 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index b77d8b78a2d..b96fb2df5f7 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -112,6 +112,7 @@ jobs: # if: ${{ github.repository_owner == 'IQSS' }} outputs: supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} + rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} steps: - name: Checkout and Setup Maven uses: IQSS/dataverse/.github/actions/setup-maven@develop @@ -140,18 +141,25 @@ jobs: DEVELOPMENT_BRANCH=${{ needs.discover.outputs.develop-branch }} .github/workflows/scripts/containers/maintain-application.sh ${{ needs.discover.outputs.branches }} + #config-image: TODO! + hub-description: name: Push description to DockerHub runs-on: ubuntu-latest permissions: contents: read packages: read - needs: base-image + needs: + - base-image + - application-image + # - config-image steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Render README - id: render + + ### BASE IMAGE + - name: Render README for base image + if: toJSON(needs.base-image.outputs.rebuilt_images) != '[]' run: | TAGS_JSON='${{ needs.base-image.outputs.supported_tag_matrix }}' echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | @@ -160,17 +168,66 @@ jobs: "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-base/src/main/docker/Dockerfile)," \ "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-base/src/backports/${branch}))" \ - | tee -a "${GITHUB_WORKSPACE}/tags.md" + | tee -a "${GITHUB_WORKSPACE}/tags-base.md" done - sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags.md" "./modules/container-base/README.md" + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-base.md" "./modules/container-base/README.md" cat "./modules/container-base/README.md" - - - name: Push description to DockerHub - if: ${{ ! inputs.dry_run && ! inputs.damp_run }} + - name: Push description to DockerHub for base image + if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.base-image.outputs.rebuilt_images) != '[]' }} uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} repository: gdcc/base short-description: "Dataverse Base Container image providing Payara application server and optimized configuration" - readme-filepath: ./modules/container-base/README.md \ No newline at end of file + readme-filepath: ./modules/container-base/README.md + + ### APPLICATION IMAGE + - name: Render README for application image + if: toJSON(needs.application-image.outputs.rebuilt_images) != '[]' + run: | + TAGS_JSON='${{ needs.application-image.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/src/main/docker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/src/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags-app.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-app.md" "./src/main/docker/README.md" + cat "./src/main/docker/README.md" + - name: Push description to DockerHub for application image + if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.application-image.outputs.rebuilt_images) != '[]' }} + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/dataverse + short-description: "Dataverse Application Container Image providing the executable" + readme-filepath: ./src/main/docker/README.md + + ### CONFIGBAKER IMAGE + #- name: Render README for config baker image + # if: toJSON(needs.config-image.outputs.rebuilt_images) != '[]' + # run: | + # TAGS_JSON='${{ needs.config-image.outputs.supported_tag_matrix }}' + # echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + # while IFS= read -r branch; do + # echo \ + # "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + # "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-configbaker/src/main/docker/Dockerfile)," \ + # "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-configbaker/src/backports/${branch}))" \ + # | tee -a "${GITHUB_WORKSPACE}/tags-config.md" + # done + # sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-config.md" "./modules/container-configbaker/README.md" + # cat "./modules/container-configbaker/README.md" + #- name: Push description to DockerHub for config baker image + # if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.config-image.outputs.rebuilt_images) != '[]' }} + # uses: peter-evans/dockerhub-description@v4 + # with: + # username: ${{ secrets.DOCKERHUB_USERNAME }} + # password: ${{ secrets.DOCKERHUB_TOKEN }} + # repository: gdcc/base + # short-description: "Dataverse Config Baker Container Image providing setup tooling and more" + # readme-filepath: ./modules/container-configbaker/README.md From 5aedb1aacaf94a5f98686ebccddc2e407b0d5474 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 8 May 2025 12:45:51 +0200 Subject: [PATCH 28/66] doc(ct): fix obsolete sentence from base image hub description --- modules/container-base/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/container-base/README.md b/modules/container-base/README.md index 0598d709eac..f6854482073 100644 --- a/modules/container-base/README.md +++ b/modules/container-base/README.md @@ -32,7 +32,6 @@ to ask for help and guidance. This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-base) happens there (again, by the community). -Community-supported image tags are based on the two most important branches: Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). For more detailed information about our tagging policy, please read about our [base image tags](https://guides.dataverse.org/en/latest/container/base-image.html#supported-image-tags) in the Dataverse Containers Guide. From a9e4a88b804a7d6512d29594ebff8e40001e64ec Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 9 May 2025 09:53:58 +0200 Subject: [PATCH 29/66] ci(ct): remove superfluous MASTER_BRANCH_TAG from app push workflow The MASTER_BRANCH_TAG variable was not being used and has been removed to simplify the workflow file. This ensures a cleaner and more maintainable configuration. --- .github/workflows/container_app_push.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 44a67ce86fa..0472ab97dee 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -24,7 +24,6 @@ env: IMAGE_TAG: unstable REGISTRY: "" # Empty means default to Docker Hub PLATFORMS: "linux/amd64,linux/arm64" - MASTER_BRANCH_TAG: alpha jobs: build: From 659a9772cd58f84b0c8d9b08a02afcc13abb54c1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 9 May 2025 09:56:49 +0200 Subject: [PATCH 30/66] build(ct): enhance app image labels with base image reference and correct version - We want to have a reference on which base image an application images builds on. For release branch app images this will be an immutable, fixed tag base image. This should make debugging things easier. - Now using app.image.version instead of project.version so the preview imaged contain the right version (minor+1) --- pom.xml | 2 ++ src/main/docker/Dockerfile | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 29c61cc7635..e4ec9b4ca92 100644 --- a/pom.xml +++ b/pom.xml @@ -1141,6 +1141,8 @@ Dockerfile ${base.image} + + ${app.image.version} @ diff --git a/src/main/docker/Dockerfile b/src/main/docker/Dockerfile index ed670294873..beb1de53cd5 100644 --- a/src/main/docker/Dockerfile +++ b/src/main/docker/Dockerfile @@ -46,16 +46,19 @@ RUN ln -s "${DEPLOY_DIR}/dataverse/supplements/jhove.conf" "${PAYARA_DIR}/glassf ln -s "${DEPLOY_DIR}/dataverse/supplements/jhoveConfig.xsd" "${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}/config/jhoveConfig.xsd" && \ sed -i "${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}/config/jhove.conf" -e "s:/usr/local/payara./glassfish/domains/domain1:${PAYARA_DIR}/glassfish/domains/${DOMAIN_NAME}:g" +# Workaround for fabric8io/docker-maven-plugin#1865 +ARG APP_IMAGE_VERSION LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.authors="Research Data Management at FZJ " \ org.opencontainers.image.url="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.documentation="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.source="https://github.com/IQSS/dataverse" \ - org.opencontainers.image.version="@project.version@" \ + org.opencontainers.image.version="$APP_IMAGE_VERSION" \ org.opencontainers.image.revision="@git.commit.id.abbrev@" \ org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.title="Dataverse Application Image" \ org.opencontainers.image.description="This container image provides the research data repository software Dataverse in a box." \ + org.opencontainers.image.base.name="$BASE_IMAGE" \ org.dataverse.deps.postgresql.version="@postgresql.server.version@" \ org.dataverse.deps.solr.version="@solr.version@" \ No newline at end of file From 85d2f0460e1d5072c3063517293ba6876bd9eafd Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 9 May 2025 12:55:53 +0200 Subject: [PATCH 31/66] docs(ct): add and extend supported image tags and add version string replacment Expanded documentation for all container image supported tags maintenance, tagging structure of app image, and development practices. Introduced substitution support for version and nextVersion in docs. Had to update and add Sphinx dependencies for this. --- doc/sphinx-guides/requirements.txt | 5 +- doc/sphinx-guides/source/conf.py | 8 ++- .../source/container/app-image.rst | 67 +++++++++++++++++-- .../source/container/base-image.rst | 12 ++-- 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/doc/sphinx-guides/requirements.txt b/doc/sphinx-guides/requirements.txt index 8eadb843cff..9c74ed75f6d 100755 --- a/doc/sphinx-guides/requirements.txt +++ b/doc/sphinx-guides/requirements.txt @@ -1,4 +1,4 @@ -Sphinx==7.2.6 +Sphinx==7.4.0 # inline icons sphinx-icon==0.1.2 @@ -11,3 +11,6 @@ sphinx-tabs==3.4.5 # jQuery sphinxcontrib-jquery + +Sphinx-Substitution-Extensions==2025.1.2 +semver>=3,<4 \ No newline at end of file diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index 26e71672f2e..eeba70dacde 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -15,6 +15,7 @@ import sys import os from datetime import datetime +import semver sys.path.insert(0, os.path.abspath('../../')) import sphinx_bootstrap_theme @@ -45,6 +46,7 @@ 'sphinxcontrib.jquery', 'myst_parser', 'sphinx_tabs.tabs', + 'sphinx_substitution_extensions', ] # Add any paths that contain templates here, relative to this directory. @@ -70,7 +72,7 @@ # The short X.Y version. version = '6.6' # The full version, including alpha/beta/rc tags. -release = '6.6' +release = version # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -437,4 +439,6 @@ rst_prolog = """ .. |toctitle| replace:: Contents: .. |anotherSub| replace:: Yes, there can be multiple. -""" +.. |version| replace:: %s +.. |nextVersion| replace:: %s +""" % (version, "%s.%s" % semver.Version.parse(version, optional_minor_and_patch=True).bump_minor().to_tuple()[0:2] ) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index e1f5517629d..41012fb6dfd 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -29,13 +29,68 @@ Supported Image Tags This image is sourced from the main upstream code `repository of the Dataverse software `_. Development and maintenance of the `image's code `_ -happens there (again, by the community). Community-supported image tags are based on the two most important -upstream branches: +happens there (again, by the community). + +All supported images are signed up for scheduled maintenance, executed every Sunday. +New revisions are kept to a minimum, usually created when some dependency needs (security) updates. +For the application images it correlates mostly to the :doc:`base image ` receiving updates. + +Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: + +- rolling: images change over time +- immutable: images are fixed and never change + +In the tags below you'll see the term "flavor". This refers to flavor of Linux the container is built on. We use Ubuntu as the basis for our images and, for the time being, the only operating system flavors we use and support are ``noble`` (6.4+) and ``jammy`` (pre-6.4). + +You can find all the tags at https://hub.docker.com/r/gdcc/dataverse/tags + +Tags for Production Use +^^^^^^^^^^^^^^^^^^^^^^^ + +The images of the three latest releases of the Dataverse project will receive updates such as security patches for the underlying operating system. +Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). +Expect disruptive changes in case of high risk security threats. + +- | **Latest** + | Definition: ``latest`` + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. +- | **Rolling Production** + | Definition: ``.-`` + | Example: :substitution-code:`|version|-noble` + | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. +- | **Immutable Production** + | Definition: ``.--r`` + | Example: :substitution-code:`|version|-noble-r1` + | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. + | This image should be especially attractive if you want explict control over when your images are updated. + +Tags for Development Use +^^^^^^^^^^^^^^^^^^^^^^^^ + +All of the tags below are strongly recommended for development purposes only due to their fast changing nature. +In addition to updates due to PR merges, the most recent are undergoing scheduled maintenance to ensure timely security fixes. +When a development cycle of the Dataverse project finishes, maintenance ceases for any tags carrying version numbers. +For now, stale images will be kept on Docker Hub indefinitely. + +- | **Unstable** + | Definition: ``unstable`` + | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) + | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. +- | **Upcoming** + | Definition: ``.-`` + | Example: :substitution-code:`|nextVersion|-noble` + | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. + Will roll over to the rolling production tag after a Dataverse release. +- | **Flexible Stack** + | Definition: ``.--p-j`` + | Example: :substitution-code:`|nextVersion|-noble-p6.2025.3-j17` + | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). + +**NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. +Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. +The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. +See also: :doc:`/developers/making-releases`. -- The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged. - (`Dockerfile `__) -- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from. - (`Dockerfile `__) Image Contents ++++++++++++++ diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index a0852a5465f..ac560fe2e7e 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -30,6 +30,10 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). +All supported images are signed up for scheduled maintenance, executed every Sunday. +New revisions are kept to a minimum, usually created when some dependency needs (security) updates. +(Examples: JRE patch releases, ImageMagick fixes, etc.) + Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: - rolling: images change over time @@ -51,11 +55,11 @@ Expect disruptive changes in case of high risk security threats. | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. - | **Rolling Production** | Definition: ``.-`` - | Example: ``6.4-noble`` + | Example: :substitution-code:`|version|-noble` | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. - | **Immutable Production** | Definition: ``.--r`` - | Example: ``6.4-noble-r1`` + | Example: :substitution-code:`|version|-noble-r1` | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. | This image should be especially attractive if you want explict control over when your images are updated. @@ -73,12 +77,12 @@ For now, stale images will be kept on Docker Hub indefinitely. | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. - | **Upcoming** | Definition: ``.-`` - | Example: ``6.5-noble`` + | Example: :substitution-code:`|nextVersion|-noble` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. - | **Flexible Stack** | Definition: ``.--p-j`` - | Example: ``6.5-noble-p6.2024.6-j17`` + | Example: :substitution-code:`|nextVersion|-noble-p6.2025.3-j17` | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). **NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. From bb4f7114f00e0126501b6c846f1f4d0e7f4e6958 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 15 May 2025 11:52:51 +0200 Subject: [PATCH 32/66] chore(ct): upgrade configbaker to Alpine 3.21 and pin down dependencies To avoid too many surprises from updates, we pin down the installed package versions. Updates will be handled by letting Renovate create PRs to update the versions. --- modules/container-configbaker/Dockerfile | 40 +++++++++++++++++++----- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 351425a17ba..0a6c2e5064b 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -9,11 +9,8 @@ ARG SOLR_VERSION # We simply have this intermediate stage here without any activity to copy the default configset over FROM solr:${SOLR_VERSION} AS solr -# Let's build us a baker -# WARNING: -# Do not upgrade the tag to :3 or :3.19 until https://pkgs.alpinelinux.org/package/v3.19/main/x86_64/c-ares is at v1.26.0+! -# See https://github.com/IQSS/dataverse/issues/10413 for more information. -FROM alpine:3.18 +# Let's build ourselves a baker +FROM alpine:3.21 ENV SCRIPT_DIR="/scripts" \ SECRETS_DIR="/secrets" \ @@ -21,11 +18,40 @@ ENV SCRIPT_DIR="/scripts" \ ENV PATH="${PATH}:${SCRIPT_DIR}" \ BOOTSTRAP_DIR="${SCRIPT_DIR}/bootstrap" -ARG APK_PACKAGES="curl bind-tools netcat-openbsd jq bash dumb-init wait4x ed postgresql-client aws-cli" +# renovate: datasource=repology depName=alpine_3_21/aws-cli +ENV AWS_CLI_VERSION="2.22.10-r0" +# renovate: datasource=repology depName=alpine_3_21/bash +ENV BASH_VERSION="5.2.37-r0" +# renovate: datasource=repology depName=alpine_3_21/bind-tools +ENV BIND_TOOLS_VERSION="9.18.36-r0" +# renovate: datasource=repology depName=alpine_3_21/curl +ENV CURL_VERSION="8.12.1-r1" +# renovate: datasource=repology depName=alpine_3_21/dumb-init +ENV DUMB_INIT_VERSION="1.2.5-r3" +# renovate: datasource=repology depName=alpine_3_21/ed +ENV ED_VERSION="1.20.2-r0" +# renovate: datasource=repology depName=alpine_3_21/jq +ENV JQ_VERSION="1.7.1-r0" +# renovate: datasource=repology depName=alpine_3_21/netcat-openbsd +ENV NETCAT_VERSION="1.226.1.1-r0" +# renovate: datasource=repology depName=alpine_3_21/postgresql17-client +ENV PGCLIENT17_VERSION="17.5-r0" +# renovate: datasource=repology depName=alpine_3_21/wait4x +ENV WAIT4X_VERSION="2.14.0-r10" RUN true && \ # Install necessary software and tools - apk add --no-cache ${APK_PACKAGES} && \ + apk add --no-cache \ + aws-cli=${AWS_CLI_VERSION} \ + bind-tools=${BIND_TOOLS_VERSION} \ + bash=${BASH_VERSION} \ + curl=${CURL_VERSION} \ + dumb-init=${DUMB_INIT_VERSION} \ + ed=${ED_VERSION} \ + jq=${JQ_VERSION} \ + netcat-openbsd=${NETCAT_VERSION} \ + postgresql17-client=${PGCLIENT17_VERSION} \ + wait4x=${WAIT4X_VERSION} && \ # Make our working directories mkdir -p ${SCRIPT_DIR} ${SECRETS_DIR} ${SOLR_TEMPLATE} From 213e28d680d0c6fa24fbf9bf72eff94e1c868d40 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 15 May 2025 11:54:21 +0200 Subject: [PATCH 33/66] doc: introduce CODEOWNERS Using this file, we can automatically assign people to review certain parts of our code. For now, we'll add the container related things. This will also be used to let Renovate know who should be assigned to PRs regarding containers. --- .github/CODEOWNERS | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000000..5c9ad7581f8 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,8 @@ + +# Any container related stuff should be assigned to / reviewed by Oliver and/or Phil +modules/container-configbaker/** @poikilotherm @pdurbin +modules/container-base/** @poikilotherm @pdurbin +src/main/docker/** @poikilotherm @pdurbin +docker-compose-dev.yml @poikilotherm @pdurbin +.github/workflows/scripts/containers** @poikilotherm @pdurbin +.github/workflows/container_* @poikilotherm @pdurbin From f71542bc602d4dbf5ff86a212f32d31830d38802 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 15 May 2025 11:55:59 +0200 Subject: [PATCH 34/66] feat: add Renovate config For now, this will only look at explicitly defined packages in defined locations. This keeps down the noise generated from the dependency bot. --- renovate.json5 | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 renovate.json5 diff --git a/renovate.json5 b/renovate.json5 new file mode 100644 index 00000000000..164aaa24e6f --- /dev/null +++ b/renovate.json5 @@ -0,0 +1,52 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + // We don't use this for now, as we want to avoid the noise - we'll only selectively activate updates + // "extends": [ + // "config:recommended", + // ] + + // We want a nice overview of dependencies under renovate's supervision + dependencyDashboard: true, + // See .github/CODEOWNERS + assigneesFromCodeOwners: true, + // Do not separate patch and minor upgrades into separate PRs for the same dependency + separateMinorPatch: false, + // Use nicer semantic commits style for messages + semanticCommits: "enabled", + + // Don't jump the gun in case something goes awry in upstream releases + minimumReleaseAge: "3 days", + // Only have this number of PRs open at any time. We will further limit this by using grouping for packages + prConcurrentLimit: 5, + // Create PRs only on weekends to avoid noise during the week (sufficient for scheduled maintenance) + schedule: ["at 06:00 am on Sunday"], + + // Only include certain paths we actually want Renovate to take care of. + includePaths: [ + "modules/container-*/**", + ], + + packageRules: [ + { + description: "Group Alpine Package Manager dependencies for the Config Baker Container Image", + matchFileNames: ["modules/container-configbaker/**/Dockerfile"], + matchDatasources: ["repology"], + groupName: "Config Baker Container - APK packages" + }, + ], + + customManagers: [ + { + customType: "regex", + description: "Update _VERSION variables in Dockerfiles", + managerFilePatterns: [ + "/(^|/|\\.)Dockerfile$/", + "/(^|/)Dockerfile\\.[^/]*$/" + ], + matchStrings: [ + "#\\s*renovate:\\s*datasource=(?.*?) depName=(?.*?)\\sENV .*?_VERSION=\"(?.*)\"\\s" + ], + versioningTemplate: "loose" + } + ] +} \ No newline at end of file From b15681c2b5ade267562306a781bcd40f1ef4fcc5 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 15 May 2025 12:31:08 +0200 Subject: [PATCH 35/66] test(ct): check if Renovatebot correctly identifies APK updates --- modules/container-configbaker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 0a6c2e5064b..949843c915d 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -37,7 +37,7 @@ ENV NETCAT_VERSION="1.226.1.1-r0" # renovate: datasource=repology depName=alpine_3_21/postgresql17-client ENV PGCLIENT17_VERSION="17.5-r0" # renovate: datasource=repology depName=alpine_3_21/wait4x -ENV WAIT4X_VERSION="2.14.0-r10" +ENV WAIT4X_VERSION="2.14.0-r9" RUN true && \ # Install necessary software and tools From 7e8969bf31fcbb9aad6f35fcc78de82deb570b86 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 16 May 2025 16:30:32 +0200 Subject: [PATCH 36/66] build(ct): move configbaker to Ubuntu 24.04 LTS - Align image with the foundation of our usual app images. Enables reusing existing scripts to detect updates. - Move away from pinning all package versions. This won't work for most package systems, as older releases are only kept temporarily. We simply need to rely on detecting updates to them. - As Ubuntu Noble has no packages for awscli or wait4x, install them manually. - As awscli requires Python, install that and a rather lightweight package manager pipx. - Add SHELL specification to avoid any problems --- modules/container-configbaker/Dockerfile | 79 ++++++++++++++---------- 1 file changed, 45 insertions(+), 34 deletions(-) diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 949843c915d..ef7a76714b1 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -10,7 +10,7 @@ ARG SOLR_VERSION FROM solr:${SOLR_VERSION} AS solr # Let's build ourselves a baker -FROM alpine:3.21 +FROM ubuntu:24.04 ENV SCRIPT_DIR="/scripts" \ SECRETS_DIR="/secrets" \ @@ -18,42 +18,53 @@ ENV SCRIPT_DIR="/scripts" \ ENV PATH="${PATH}:${SCRIPT_DIR}" \ BOOTSTRAP_DIR="${SCRIPT_DIR}/bootstrap" -# renovate: datasource=repology depName=alpine_3_21/aws-cli -ENV AWS_CLI_VERSION="2.22.10-r0" -# renovate: datasource=repology depName=alpine_3_21/bash -ENV BASH_VERSION="5.2.37-r0" -# renovate: datasource=repology depName=alpine_3_21/bind-tools -ENV BIND_TOOLS_VERSION="9.18.36-r0" -# renovate: datasource=repology depName=alpine_3_21/curl -ENV CURL_VERSION="8.12.1-r1" -# renovate: datasource=repology depName=alpine_3_21/dumb-init -ENV DUMB_INIT_VERSION="1.2.5-r3" -# renovate: datasource=repology depName=alpine_3_21/ed -ENV ED_VERSION="1.20.2-r0" -# renovate: datasource=repology depName=alpine_3_21/jq -ENV JQ_VERSION="1.7.1-r0" -# renovate: datasource=repology depName=alpine_3_21/netcat-openbsd -ENV NETCAT_VERSION="1.226.1.1-r0" -# renovate: datasource=repology depName=alpine_3_21/postgresql17-client -ENV PGCLIENT17_VERSION="17.5-r0" -# renovate: datasource=repology depName=alpine_3_21/wait4x -ENV WAIT4X_VERSION="2.14.0-r9" +ARG PKGS="curl dnsutils dumb-init ed jq netcat-openbsd postgresql-client" +# renovate: datasource=github-releases depName=wait4x/wait4x +ARG WAIT4X_VERSION="v3.3.0" +# renovate: datasource=pypi depName=awscli +ARG AWSCLI_VERSION="1.40.15" +ARG PYTHON_PKGS="awscli==${AWSCLI_VERSION}" + +# Auto-populated by BuildKit / buildx +ARG TARGETARCH +SHELL ["/bin/bash", "-eu", "-c"] RUN true && \ - # Install necessary software and tools - apk add --no-cache \ - aws-cli=${AWS_CLI_VERSION} \ - bind-tools=${BIND_TOOLS_VERSION} \ - bash=${BASH_VERSION} \ - curl=${CURL_VERSION} \ - dumb-init=${DUMB_INIT_VERSION} \ - ed=${ED_VERSION} \ - jq=${JQ_VERSION} \ - netcat-openbsd=${NETCAT_VERSION} \ - postgresql17-client=${PGCLIENT17_VERSION} \ - wait4x=${WAIT4X_VERSION} && \ # Make our working directories - mkdir -p ${SCRIPT_DIR} ${SECRETS_DIR} ${SOLR_TEMPLATE} + mkdir -p ${SCRIPT_DIR} ${SECRETS_DIR} ${SOLR_TEMPLATE} && \ + + # Install packages + apt-get update -q && \ + apt-get install -qqy --no-install-recommends ${PKGS} && \ + + # Workaround to install Python and pipx 1.5+ on Ubuntu 24.04 LTS: first install Python and pipx 1.4 \ + # Adapted from https://github.com/pypa/pipx/issues/1481#issuecomment-2593124603 + apt -qqy --no-install-recommends install python3 python3-venv pipx && \ + # Now install 1.5+ in ~/.local/bin/ + pipx install pipx && \ + # Remove 1.4 again + apt purge -qqy --autoremove pipx && \ + # Install 1.5+ in /usr/local/bin/pipx + ~/.local/bin/pipx install --global pipx && \ + # Remove the virtual env install of pipx + rm -rf "~/.local" && \ + + # Cleanup apt cache + rm -rf "/var/lib/apt/lists/*" + +# New step (and shell) as this is a different manager and we need pipx around as command +RUN true && \ + # Install things not available as packages \ + ARCH="${TARGETARCH:-$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')}" && \ + + # 1. wait4x \ + curl -sSfL -o /usr/bin/wait4x.tar.gz "https://github.com/wait4x/wait4x/releases/download/${WAIT4X_VERSION}/wait4x-linux-${ARCH}.tar.gz" && \ + curl -sSfL -o /tmp/w4x-checksum "https://github.com/wait4x/wait4x/releases/download/${WAIT4X_VERSION}/wait4x-linux-${ARCH}.tar.gz.sha256sum" && \ + echo "$(cat /tmp/w4x-checksum | cut -f1 -d" ") /usr/bin/wait4x.tar.gz" | sha256sum -c - && \ + tar -xzf /usr/bin/wait4x.tar.gz -C /usr/bin && chmod +x /usr/bin/wait4x && \ + + # 2. Python packages + pipx install --global ${PYTHON_PKGS} # Get in the scripts COPY maven/scripts maven/solr/update-fields.sh ${SCRIPT_DIR}/ From 5762cdf432fbff27bd38e08657cf1c17ff75cbdb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 16 May 2025 16:31:03 +0200 Subject: [PATCH 37/66] chore(renovate): improve Renovate config for better package grouping and parsing Refined grouping rules for Config Baker container image dependencies and enhanced regex for version extraction. Adjusted default versioning template to prioritize semantic versioning when unspecified. These changes ensure more accurate and maintainable update tracking. --- renovate.json5 | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/renovate.json5 b/renovate.json5 index 164aaa24e6f..2d7875c036e 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -28,10 +28,9 @@ packageRules: [ { - description: "Group Alpine Package Manager dependencies for the Config Baker Container Image", - matchFileNames: ["modules/container-configbaker/**/Dockerfile"], - matchDatasources: ["repology"], - groupName: "Config Baker Container - APK packages" + description: "Group package updates for the Config Baker Container Image", + matchFileNames: ["modules/container-configbaker/**/*[dD]ockerfile"], + groupName: "Config Baker Container Packages" }, ], @@ -44,9 +43,9 @@ "/(^|/)Dockerfile\\.[^/]*$/" ], matchStrings: [ - "#\\s*renovate:\\s*datasource=(?.*?) depName=(?.*?)\\sENV .*?_VERSION=\"(?.*)\"\\s" + "# renovate: datasource=(?[a-zA-Z0-9-._]+?) depName=(?[^\\s]+?)(?: (lookupName|packageName)=(?[^\\s]+?))?(?: versioning=(?[^\\s]+?))?(?: extractVersion=(?[^\\s]+?))?(?: registryUrl=(?[^\\s]+?))?\\s(?:ENV|ARG)\\s+[A-Za-z0-9_]+?_VERSION[ =][\"']?(?.+?)[\"']?\\s" ], - versioningTemplate: "loose" + versioningTemplate: '{{#if versioning}}{{{versioning}}}{{else}}semver{{/if}}' } ] } \ No newline at end of file From 1a3395be0e5404a38015390815b5526f57284897 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 16 May 2025 16:34:30 +0200 Subject: [PATCH 38/66] test(ct): downgrade wait4x to test renovate detection --- modules/container-configbaker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index ef7a76714b1..d52e717603b 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -20,7 +20,7 @@ ENV PATH="${PATH}:${SCRIPT_DIR}" \ ARG PKGS="curl dnsutils dumb-init ed jq netcat-openbsd postgresql-client" # renovate: datasource=github-releases depName=wait4x/wait4x -ARG WAIT4X_VERSION="v3.3.0" +ARG WAIT4X_VERSION="v3.2.0" # renovate: datasource=pypi depName=awscli ARG AWSCLI_VERSION="1.40.15" ARG PYTHON_PKGS="awscli==${AWSCLI_VERSION}" From d04311fd3751a758a68128d646e82d181a08dda3 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Fri, 16 May 2025 17:16:23 +0200 Subject: [PATCH 39/66] chore: disable Renovate managers for Maven and Dockerfiles We don't want that noise at the moment. --- renovate.json5 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/renovate.json5 b/renovate.json5 index 2d7875c036e..063c7b41cc7 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -27,6 +27,16 @@ ], packageRules: [ + { + description: "Disable Maven Manager", + matchManagers: "maven", + enabled: false + }, + { + description: "Disable Dockerfile Manager", + matchManagers: "dockerfile", + enabled: false + }, { description: "Group package updates for the Config Baker Container Image", matchFileNames: ["modules/container-configbaker/**/*[dD]ockerfile"], From 4c4d98c3f216ac4e07c01680571991618dc63cd7 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 15:50:46 +0200 Subject: [PATCH 40/66] ci(docs): update Sphinx workflow to use `sphinx-notes/pages` action Replaced `uncch-rdmc/sphinx-action` with `sphinx-notes/pages` for better compatibility and flexibility (Python & Sphinx versions!!!). Added dynamic Sphinx version lookup and updated paths and options to align with the new action requirements. --- .github/workflows/guides_build_sphinx.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml index fa3a876c418..a3b5882626c 100644 --- a/.github/workflows/guides_build_sphinx.yml +++ b/.github/workflows/guides_build_sphinx.yml @@ -11,6 +11,18 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: uncch-rdmc/sphinx-action@master + - id: lookup + run: | + echo "sphinx_version=$(grep Sphinx== ./doc/sphinx-guides/requirements.txt | tr -s "=" | cut -f 2 -d=)" | tee -a "${GITHUB_OUTPUT}" + - run: | + sudo apt-get update -q + sudo apt-get install -qqy --no-install-recommends graphviz + - uses: sphinx-notes/pages@v3 with: - docs-folder: "doc/sphinx-guides/" + documentation_path: ./doc/sphinx-guides/source + requirements_path: ./doc/sphinx-guides/requirements.txt + sphinx_version: ${{ steps.lookup.outputs.sphinx_version }} + sphinx_build_options: "-W" + cache: false + publish: false + From b6a429e30a08acee7f5c759e654fee821e657124 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 16:20:52 +0200 Subject: [PATCH 41/66] ci(ct): add Trivy scan utility for checking OS vulnerability fixes Introduce `check_trivy_fixes_for_os` to analyze container images for fixable OS vulnerabilities using Trivy. Also add a helper function, `is_bin_in_path`, to ensure required binaries are available. --- .github/workflows/scripts/containers/utils.sh | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/scripts/containers/utils.sh b/.github/workflows/scripts/containers/utils.sh index aac317464ad..a61df5467fe 100644 --- a/.github/workflows/scripts/containers/utils.sh +++ b/.github/workflows/scripts/containers/utils.sh @@ -2,6 +2,10 @@ set -euo pipefail +function is_bin_in_path { + builtin type -P "$1" &> /dev/null +} + function check_newer_parent() { PARENT_IMAGE="$1" # Get namespace, default to "library" if not found @@ -61,6 +65,27 @@ function check_newer_pkgs() { } +function check_trivy_fixes_for_os() { + IMAGE_REF="$1" + if [[ -z "$IMAGE_REF" ]]; then + echo "You must give an image reference as argument to check_trivy_fixes_for_os" + exit 1 + fi + is_bin_in_path trivy || { echo "Trivy Scanner not installed" 1>&2; exit 1; } + JSON_REPORT=$(mktemp) + + trivy image --ignore-unfixed --scanners vuln --disable-telemetry --pkg-types os -f json "$IMAGE_REF" -o "$JSON_REPORT" + + HAS_FIXES=$( jq -r '.Results[] | select(has("Vulnerabilities") and .Vulnerabilities != null and (.Vulnerabilities | length > 100)) | .Vulnerabilities | length > 0' "$JSON_REPORT") + if [[ "true" = "$HAS_FIXES" ]]; then + echo "Trivy Scan showed fixes to known vulnerabilities by updating packages exist for image $IMAGE_REF" + return 0 + else + echo "Trivy Scan showed no fixes to known vulnerabilities by updating packages exist for image $IMAGE_REF" + return 1 + fi +} + function current_revision() { IMAGE="$1" IMAGE_NS_REPO="${IMAGE%:*}" From 328bee3817722e9df1055cebd1314f6b92b7bb06 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 16:23:11 +0200 Subject: [PATCH 42/66] chore,ci(ct): update Renovate schedule to use cron syntax Replaced the fixed Sunday 6:00 AM schedule with a cron expression as recommended by the docs. This maintains the behavior of creating PRs only on Sundays, but hopefully works correctly now. Looking at the logs within the Mend Renovate Board, the general scanning etc works but the rule seems to not allow execution of PR creation for some reason. Educated guess: it's not 6 am when the scheduler runs and the logic is to exclusive. --- renovate.json5 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renovate.json5 b/renovate.json5 index 063c7b41cc7..c7bc73cef49 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -18,8 +18,8 @@ minimumReleaseAge: "3 days", // Only have this number of PRs open at any time. We will further limit this by using grouping for packages prConcurrentLimit: 5, - // Create PRs only on weekends to avoid noise during the week (sufficient for scheduled maintenance) - schedule: ["at 06:00 am on Sunday"], + // By default, create PRs only on Sundays to avoid noise during the week (sufficient for scheduled maintenance) + schedule: ["* * * * 0"], // Only include certain paths we actually want Renovate to take care of. includePaths: [ From 795a9cbdd87458d8dc6296fcfb931dc599340648 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 16:23:55 +0200 Subject: [PATCH 43/66] feat(ct): set configurable base image for configbaker Introduce a `BASE_IMAGE` argument to make the base image configurable for the configbaker Dockerfile. Updated the POM to define default values and streamline tag inheritance for development and maintenance scenarios. --- modules/container-configbaker/Dockerfile | 7 +++++-- pom.xml | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index d52e717603b..328ae0fab55 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -6,11 +6,13 @@ # This build arg must be given or build will fail ARG SOLR_VERSION +ARG BASE_IMAGE="ubuntu:noble" + # We simply have this intermediate stage here without any activity to copy the default configset over FROM solr:${SOLR_VERSION} AS solr # Let's build ourselves a baker -FROM ubuntu:24.04 +FROM ${BASE_IMAGE} ENV SCRIPT_DIR="/scripts" \ SECRETS_DIR="/secrets" \ @@ -95,4 +97,5 @@ LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.title="Dataverse Config Baker Image" \ - org.opencontainers.image.description="This container image configures Dataverse and provides other tooling" + org.opencontainers.image.description="This container image configures Dataverse and provides other tooling" \ + org.opencontainers.image.base.name="$BASE_IMAGE" diff --git a/pom.xml b/pom.xml index e4ec9b4ca92..537ac396559 100644 --- a/pom.xml +++ b/pom.xml @@ -1075,6 +1075,7 @@ unstable false @@ -1089,7 +1090,10 @@ -p${payara.version}-j${target.java.version} gdcc/configbaker:${conf.image.tag} + ${app.image.tag} + noble + ubuntu:${conf.image.flavor} false @@ -1172,6 +1176,7 @@ ${project.basedir}/modules/container-configbaker/Dockerfile + ${conf.image.base} ${SOLR_VERSION} @ From 3b41d84dcceb5131b24542312b2c806dab737887 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 16:24:30 +0200 Subject: [PATCH 44/66] feat(ct): add backports for container-configbaker to support image customization Introduces patches to parent POM files for v6.4, v6.5, and v6.6 backports. Updates include better handling of image versioning, configuration, and build options to enhance development and maintenance workflows. --- .../backports/v6.4/001-parent-pom.xml.patch | 10 ++ .../backports/v6.4/002-pom.xml.patch | 107 ++++++++++++++++++ .../backports/v6.5/001-parent-pom.xml.patch | 10 ++ .../backports/v6.5/002-pom.xml.patch | 107 ++++++++++++++++++ .../backports/v6.6/001-parent-pom.xml.patch | 10 ++ .../backports/v6.6/002-pom.xml.patch | 107 ++++++++++++++++++ 6 files changed, 351 insertions(+) create mode 100644 modules/container-configbaker/backports/v6.4/001-parent-pom.xml.patch create mode 100644 modules/container-configbaker/backports/v6.4/002-pom.xml.patch create mode 100644 modules/container-configbaker/backports/v6.5/001-parent-pom.xml.patch create mode 100644 modules/container-configbaker/backports/v6.5/002-pom.xml.patch create mode 100644 modules/container-configbaker/backports/v6.6/001-parent-pom.xml.patch create mode 100644 modules/container-configbaker/backports/v6.6/002-pom.xml.patch diff --git a/modules/container-configbaker/backports/v6.4/001-parent-pom.xml.patch b/modules/container-configbaker/backports/v6.4/001-parent-pom.xml.patch new file mode 100644 index 00000000000..9b2b9dd4b5c --- /dev/null +++ b/modules/container-configbaker/backports/v6.4/001-parent-pom.xml.patch @@ -0,0 +1,10 @@ +--- a/modules/dataverse-parent/pom.xml ++++ b/modules/dataverse-parent/pom.xml +@@ -455,6 +455,7 @@ + --> + + ${revision} ++ ${base.image.version} + + + 17 + + gdcc/dataverse:${app.image.tag} ++ + unstable ++ false + false ++ + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} ++ + gdcc/configbaker:${conf.image.tag} ++ + ${app.image.tag} +- ++ alpine ++ alpine:3.18 ++ false ++ + +- ++ + + ${app.image} + ${postgresql.server.version} +@@ -1088,7 +1102,7 @@ + dataverse + ${app.skipDeploy} + +- ++ + + + +@@ -1106,7 +1120,7 @@ + + + +- ++ + + + io.fabric8 +@@ -1119,6 +1133,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1133,21 +1148,22 @@ + assembly.xml + + +- ++ + + +- ++ + + compose + ${project.basedir} + docker-compose-dev.yml + + +- ++ + + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} +@@ -1155,6 +1171,7 @@ + + ${project.basedir}/modules/container-configbaker/Dockerfile + ++ ${conf.image.base} + ${SOLR_VERSION} + + @ +@@ -1162,7 +1179,7 @@ + ${project.basedir}/modules/container-configbaker/assembly.xml + + +- ++ + + + ${revision} ++ ${base.image.version} + + + 17 + + gdcc/dataverse:${app.image.tag} ++ + unstable ++ false + false ++ + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} ++ + gdcc/configbaker:${conf.image.tag} ++ + ${app.image.tag} +- ++ alpine ++ alpine:3.18 ++ false ++ + +- ++ + + ${app.image} + ${postgresql.server.version} +@@ -1088,7 +1102,7 @@ + dataverse + ${app.skipDeploy} + +- ++ + + + +@@ -1106,7 +1120,7 @@ + + + +- ++ + + + io.fabric8 +@@ -1119,6 +1133,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1133,21 +1148,22 @@ + assembly.xml + + +- ++ + + +- ++ + + compose + ${project.basedir} + docker-compose-dev.yml + + +- ++ + + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} +@@ -1155,6 +1171,7 @@ + + ${project.basedir}/modules/container-configbaker/Dockerfile + ++ ${conf.image.base} + ${SOLR_VERSION} + + @ +@@ -1162,7 +1179,7 @@ + ${project.basedir}/modules/container-configbaker/assembly.xml + + +- ++ + + + ${revision} ++ ${base.image.version} + + + 17 + + gdcc/dataverse:${app.image.tag} ++ + unstable ++ false + false ++ + gdcc/base:${base.image.tag} + + noble + +- ${base.image.version}-${base.image.flavor}-p${payara.version}-j${target.java.version} ++ ${base.image.version}-${base.image.flavor}${base.image.tag.suffix} ++ ++ -p${payara.version}-j${target.java.version} ++ + gdcc/configbaker:${conf.image.tag} ++ + ${app.image.tag} +- ++ alpine ++ alpine:3.18 ++ false ++ + +- ++ + + ${app.image} + ${postgresql.server.version} +@@ -1088,7 +1102,7 @@ + dataverse + ${app.skipDeploy} + +- ++ + + + +@@ -1106,7 +1120,7 @@ + + + +- ++ + + + io.fabric8 +@@ -1119,6 +1133,7 @@ + dev_dataverse + ${app.image} + ++ ${app.skipBuild} + + + ${docker.platforms} +@@ -1133,21 +1148,22 @@ + assembly.xml + + +- ++ + + +- ++ + + compose + ${project.basedir} + docker-compose-dev.yml + + +- ++ + + dev_bootstrap + ${conf.image} + ++ ${conf.skipBuild} + + + ${docker.platforms} +@@ -1155,6 +1171,7 @@ + + ${project.basedir}/modules/container-configbaker/Dockerfile + ++ ${conf.image.base} + ${SOLR_VERSION} + + @ +@@ -1162,7 +1179,7 @@ + ${project.basedir}/modules/container-configbaker/assembly.xml + + +- ++ + /r ${GITHUB_WORKSPACE}/tags-config.md" "./modules/container-configbaker/README.md" - # cat "./modules/container-configbaker/README.md" - #- name: Push description to DockerHub for config baker image - # if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.config-image.outputs.rebuilt_images) != '[]' }} - # uses: peter-evans/dockerhub-description@v4 - # with: - # username: ${{ secrets.DOCKERHUB_USERNAME }} - # password: ${{ secrets.DOCKERHUB_TOKEN }} - # repository: gdcc/base - # short-description: "Dataverse Config Baker Container Image providing setup tooling and more" - # readme-filepath: ./modules/container-configbaker/README.md + - name: Render README for config baker image + if: toJSON(needs.config-image.outputs.rebuilt_images) != '[]' + run: | + TAGS_JSON='${{ needs.config-image.outputs.supported_tag_matrix }}' + echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | + while IFS= read -r branch; do + echo \ + "- \`$( echo "$TAGS_JSON" | jq --arg v "$branch" -r '.[$v] | join("`, `")' )\`" \ + "([Dockerfile](https://github.com/IQSS/dataverse/blob/${branch}/modules/container-configbaker/Dockerfile)," \ + "[Patches](https://github.com/IQSS/dataverse/blob/develop/modules/container-configbaker/backports/${branch}))" \ + | tee -a "${GITHUB_WORKSPACE}/tags-config.md" + done + sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-config.md" "./modules/container-configbaker/README.md" + cat "./modules/container-configbaker/README.md" + - name: Push description to DockerHub for config baker image + if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.config-image.outputs.rebuilt_images) != '[]' }} + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: gdcc/base + short-description: "Dataverse Config Baker Container Image providing setup tooling and more" + readme-filepath: ./modules/container-configbaker/README.md diff --git a/modules/container-configbaker/README.md b/modules/container-configbaker/README.md index 17b6f985798..38118f18b6d 100644 --- a/modules/container-configbaker/README.md +++ b/modules/container-configbaker/README.md @@ -24,14 +24,28 @@ to ask for help and guidance. ## Supported Image Tags +## Supported Image Tags + This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). -Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker) -happens there (again, by the community). Community-supported image tags are based on the two most important branches: +Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker) happens there (again, by the community). + +Our tagging is inspired by [Bitnami](https://docs.vmware.com/en/VMware-Tanzu-Application-Catalog/services/tutorials/GUID-understand-rolling-tags-containers-index.html). +For more detailed information about our tagging policy, please read about our [config baker image tags](https://guides.dataverse.org/en/latest/container/configbaker-image.html#supported-image-tags) in the Dataverse Containers Guide. + +For ease of use, here is a list of images that are currently maintained. + + + +All of them are rolling tags, except those ending with `-r`, which are the most recent immutable tags. +The `unstable` tags are the current development branch snapshot. +We strongly recommend using only immutable tags for production use cases. + +Within the main repository, you may find the image's files at `/modules/container-configbaker`. +This Maven module uses the [Maven Docker Plugin](https://dmp.fabric8.io) to build and ship the image. +You may use, extend, or alter this image to your liking and/or host in some different registry if you want to. -- The `unstable` tag corresponds to the `develop` branch, where pull requests are merged. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker/src/main/docker/Dockerfile)) -- The `alpha` tag corresponds to the `master` branch, where releases are cut from. - ([`Dockerfile`](https://github.com/IQSS/dataverse/tree/master/modules/container-configbaker/src/main/docker/Dockerfile)) +**Supported architectures:** This image is created as a "multi-arch image", supporting the most common architectures +Dataverse usually runs on: AMD64 (Windows/Linux/...) and ARM64 (Apple M1/M2). ## License From f747c74b6942e67a3119e5d369213c87f9ec1298 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 16:46:46 +0200 Subject: [PATCH 47/66] docs(ct): improve configbaker image documentation with tagging details Added explanations for image maintenance, supported Linux flavors, and detailed descriptions of tagging schemes for both production and development use. This enhances clarity for users managing and selecting the appropriate container images. --- .github/workflows/container_maintenance.yml | 6 +- .../source/container/configbaker-image.rst | 65 ++++++++++++++++--- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index a2439111d2e..722e1f5fe71 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -252,9 +252,9 @@ jobs: ### CONFIGBAKER IMAGE - name: Render README for config baker image - if: toJSON(needs.config-image.outputs.rebuilt_images) != '[]' + if: toJSON(needs.configbaker-image.outputs.rebuilt_images) != '[]' run: | - TAGS_JSON='${{ needs.config-image.outputs.supported_tag_matrix }}' + TAGS_JSON='${{ needs.configbaker-image.outputs.supported_tag_matrix }}' echo "$TAGS_JSON" | jq -r 'keys | sort | reverse | .[]' | while IFS= read -r branch; do echo \ @@ -266,7 +266,7 @@ jobs: sed -i -e "/<\!-- TAG BLOCK HERE -->/r ${GITHUB_WORKSPACE}/tags-config.md" "./modules/container-configbaker/README.md" cat "./modules/container-configbaker/README.md" - name: Push description to DockerHub for config baker image - if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.config-image.outputs.rebuilt_images) != '[]' }} + if: ${{ ! inputs.dry_run && ! inputs.damp_run && toJSON(needs.configbaker-image.outputs.rebuilt_images) != '[]' }} uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst index 09e431eb547..a230be048d2 100644 --- a/doc/sphinx-guides/source/container/configbaker-image.rst +++ b/doc/sphinx-guides/source/container/configbaker-image.rst @@ -19,14 +19,63 @@ Supported Image Tags This image is sourced from the main upstream code `repository of the Dataverse software `_. Development and maintenance of the `image's code `_ -happens there (again, by the community). Community-supported image tags are based on the two most important -upstream branches: - -- The ``unstable`` tag corresponds to the ``develop`` branch, where pull requests are merged. - (`Dockerfile `__) -- The ``alpha`` tag corresponds to the ``master`` branch, where releases are cut from. - (`Dockerfile `__) - +happens there (again, by the community). + +All supported images are signed up for scheduled maintenance, executed every Sunday. +New revisions are kept to a minimum, usually created when some dependency needs (security) updates. +Be advised: the Trivy Scanner is used to check for fixed vulnerabilities and rebuilds are issued if such a fix is detected. + +Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: + +- rolling: images change over time +- immutable: images are fixed and never change + +In the tags below you'll see the term "flavor". This refers to flavor of Linux the container is built on. We use Ubuntu as the basis for our images and, for the time being, the only operating system flavors we use and support are ``noble`` (6.7+) and ``alpine`` (pre-6.7). + +You can find all the tags at https://hub.docker.com/r/gdcc/configbaker/tags + +Tags for Production Use +^^^^^^^^^^^^^^^^^^^^^^^ + +The images of the three latest releases of the Dataverse project will receive updates such as security patches for the underlying operating system. +Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). +Expect disruptive changes in case of high risk security threats. + +- | **Latest** + | Definition: ``latest`` + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. +- | **Rolling Production** + | Definition: ``.-`` + | Example: :substitution-code:`|version|-noble` + | Summary: Rolling tag, pointing to the latest revision of an immutable production image for released versions of Dataverse. +- | **Immutable Production** + | Definition: ``.--r`` + | Example: :substitution-code:`|version|-noble-r1` + | Summary: An **immutable tag** where the revision is incremented for rebuilds of the image. + | This image should be especially attractive if you want explict control over when your images are updated. + +Tags for Development Use +^^^^^^^^^^^^^^^^^^^^^^^^ + +All of the tags below are strongly recommended for development purposes only due to their fast changing nature. +In addition to updates due to PR merges, the most recent are undergoing scheduled maintenance to ensure timely security fixes. +When a development cycle of the Dataverse project finishes, maintenance ceases for any tags carrying version numbers. +For now, stale images will be kept on Docker Hub indefinitely. + +- | **Unstable** + | Definition: ``unstable`` + | Summary: Rolling tag, tracking the ``develop`` branch (see also :ref:`develop-branch`). (`Dockerfile `__) + | Please expect abrupt changes like new Payara or Java versions as well as OS updates or flavor switches when using this tag. +- | **Upcoming** + | Definition: ``.-`` + | Example: :substitution-code:`|nextVersion|-noble` + | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. + Will roll over to the rolling production tag after a Dataverse release. + +**NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. +Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. +The rolling tag in use during the cycle will be ``6.(x+1)-FFF`` and ``6.(x+1)-FFF-p6.202P.P-jJJ``. +See also: :doc:`/developers/making-releases`. Image Contents From 3beee2979cc024c14b2a5f122ccc4afbe4b5976e Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 2 Jun 2025 17:20:38 +0200 Subject: [PATCH 48/66] style(ct): use app.image.version for configbaker OCI metadata tags To have proper references also for future versions, we cannot just use project.version. (As seen with the app image Dockerfile, just transferring the method) Needs a workaround for fabric8io/docker-maven-plugin#1865: Included APP_IMAGE_VERSION in pom.xml and Dockerfile to address version handling issues with the plugin. This ensures proper image versioning and compatibility in the build process. --- modules/container-configbaker/Dockerfile | 4 +++- pom.xml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/container-configbaker/Dockerfile b/modules/container-configbaker/Dockerfile index 328ae0fab55..6f4e0c066bc 100644 --- a/modules/container-configbaker/Dockerfile +++ b/modules/container-configbaker/Dockerfile @@ -87,12 +87,14 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] # By default run a script that will print a help message and terminate CMD ["help.sh"] +# Workaround for fabric8io/docker-maven-plugin#1865 +ARG APP_IMAGE_VERSION LABEL org.opencontainers.image.created="@git.build.time@" \ org.opencontainers.image.authors="Research Data Management at FZJ " \ org.opencontainers.image.url="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.documentation="https://guides.dataverse.org/en/latest/container/" \ org.opencontainers.image.source="https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker" \ - org.opencontainers.image.version="@project.version@" \ + org.opencontainers.image.version="$APP_IMAGE_VERSION" \ org.opencontainers.image.revision="@git.commit.id.abbrev@" \ org.opencontainers.image.vendor="Global Dataverse Community Consortium" \ org.opencontainers.image.licenses="Apache-2.0" \ diff --git a/pom.xml b/pom.xml index 537ac396559..f310fee8698 100644 --- a/pom.xml +++ b/pom.xml @@ -1178,6 +1178,8 @@ ${conf.image.base} ${SOLR_VERSION} + + ${app.image.version} @ From 6ac8d311e41e956d44803150ee8b350f992ab902 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 2 Jun 2025 14:15:17 -0400 Subject: [PATCH 49/66] update docker tagging docs #10618 --- doc/sphinx-guides/source/developers/making-releases.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index ec3f6589395..b62738bda2e 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -174,15 +174,13 @@ Check for merged pull requests that have no milestone by going to https://github (Optional) Test Docker Images ----------------------------- -After the "master" branch has been updated and the GitHub Action to build and push Docker images has run (see `PR #9776 `_), go to https://hub.docker.com/u/gdcc and make sure the "alpha" tag for the following images has been updated: +After the "master" branch has been updated and the GitHub Action to build and push Docker images has run (see `PR #9776 `_), go to https://hub.docker.com/u/gdcc and make sure the "latest" tag for the following images has been updated: - https://hub.docker.com/r/gdcc/base - https://hub.docker.com/r/gdcc/dataverse - https://hub.docker.com/r/gdcc/configbaker -To test these images against our API test suite, go to the "alpha" workflow at https://github.com/gdcc/api-test-runner/actions/workflows/alpha.yml and run it. - -Don't be surprised if there are failures. The test runner is a work in progress! Additional dependencies or settings may have been added to the "develop" workflow. Copy them over and try again. +TODO: Get https://github.com/gdcc/api-test-runner working. .. _build-guides: From 2f4244a117d1512c3cd16f64d87bf19d12a651e9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 00:20:21 +0200 Subject: [PATCH 50/66] docs(ct): simplify headings in container image guides As requested by @pdurbin during review. Renamed headings for base and application image documentation to improve clarity and consistency. --- doc/sphinx-guides/source/container/app-image.rst | 4 ++-- doc/sphinx-guides/source/container/base-image.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index 41012fb6dfd..20032b5ef5e 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -1,5 +1,5 @@ -Dataverse Application Image -=========================== +Application Image +================= The application image is a layer on top of the base image and contains the Dataverse software. diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index ac560fe2e7e..454ffffaa95 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -1,5 +1,5 @@ -Application Base Image -====================== +Base Image +========== The base image contains Payara and other dependencies that the Dataverse software runs on. It is the foundation for the :doc:`app-image`. Note that some dependencies, such as PostgreSQL and Solr, run in their own containers and are not part of the base image. From 21e0440d1d10087bd5081a1b0bfd55624fa3e5ef Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 00:25:04 +0200 Subject: [PATCH 51/66] docs(ct): standardize Sphinx anchor names for supported image tag references across container guides Updated anchor references for image tag sections to ensure consistency and clarity in documentation. Adjusted links and headings accordingly for improved navigation. --- doc/sphinx-guides/source/container/app-image.rst | 2 +- doc/sphinx-guides/source/container/base-image.rst | 2 +- doc/sphinx-guides/source/container/configbaker-image.rst | 2 ++ doc/sphinx-guides/source/container/running/demo.rst | 2 +- doc/sphinx-guides/source/developers/making-releases.rst | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index 20032b5ef5e..7810e16d07c 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -22,7 +22,7 @@ IQSS will not offer you support how to deploy or run it, please reach out to the You might be interested in taking a look at :doc:`../developers/containers`, linking you to some (community-based) efforts. -.. _supported-image-tags-app: +.. _app-image-supported-tags: Supported Image Tags ++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 454ffffaa95..86deebdaa1f 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -21,7 +21,7 @@ IQSS will not offer you support how to deploy or run it, please reach out to the You might be interested in taking a look at :doc:`../developers/containers`, linking you to some (community-based) efforts. -.. _base-supported-image-tags: +.. _base-image-supported-tags: Supported Image Tags ++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst index a230be048d2..b303594ea8f 100644 --- a/doc/sphinx-guides/source/container/configbaker-image.rst +++ b/doc/sphinx-guides/source/container/configbaker-image.rst @@ -14,6 +14,8 @@ To see the Config Baker help screen: ``docker run -it --rm gdcc/configbaker:unstable`` +.. _config-image-supported-tags: + Supported Image Tags ++++++++++++++++++++ diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index 18fbc6c808d..162632acc68 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -291,7 +291,7 @@ Additional containers are used in development (see :doc:`../dev-usage`), but for Tags and Versions +++++++++++++++++ -The compose file references a tag called "alpha", which corresponds to the latest released version of Dataverse. This means that if a release of Dataverse comes out while you are demo'ing or evaluating, the version of Dataverse you are using could change if you do a ``docker pull``. We are aware that there is a desire for tags that correspond to versions to ensure consistency. You are welcome to join `the discussion `_ and otherwise get in touch (see :ref:`helping-containers`). For more on tags, see :ref:`supported-image-tags-app`. +The compose file references a tag called "alpha", which corresponds to the latest released version of Dataverse. This means that if a release of Dataverse comes out while you are demo'ing or evaluating, the version of Dataverse you are using could change if you do a ``docker pull``. We are aware that there is a desire for tags that correspond to versions to ensure consistency. You are welcome to join `the discussion `_ and otherwise get in touch (see :ref:`helping-containers`). For more on tags, see :ref:`app-image-supported-tags`. Once Dataverse is running, you can check which version you have through the normal methods: diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index b62738bda2e..14ad87ad600 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -313,7 +313,7 @@ Create a new branch (any name is fine but ``prepare-next-iteration`` is suggeste Create a pull request and put it through code review, like usual. Give it a milestone of the next release, the one **after** the one we're working on. Once the pull request has been approved, merge it. It should the the first PR merged of the next release. -For more background, see :ref:`base-supported-image-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896 +For more background, see :ref:`base-image-supported-tags`. For an example, see https://github.com/IQSS/dataverse/pull/10896 Lift the Code Freeze and Encourage Developers to Update Their Branches ---------------------------------------------------------------------- From 0c66edf16cc4d5b4f2bc654bce2469f9fb965243 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 00:25:45 +0200 Subject: [PATCH 52/66] docs(ct): remove Flexible Stack tag description from app-image guide Error correction: the flex stack options are not available for the app or config images, only for the base image! --- doc/sphinx-guides/source/container/app-image.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index 7810e16d07c..df04423f26d 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -81,10 +81,6 @@ For now, stale images will be kept on Docker Hub indefinitely. | Example: :substitution-code:`|nextVersion|-noble` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. -- | **Flexible Stack** - | Definition: ``.--p-j`` - | Example: :substitution-code:`|nextVersion|-noble-p6.2025.3-j17` - | Summary: Rolling tag during a development cycle of the Dataverse software (`Dockerfile `__). **NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. From c9d3feff77fdee08b572f217362531fe251fe8d0 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 00:27:26 +0200 Subject: [PATCH 53/66] chore(pom): bump fabric8-dmp version to 0.46.0 --- modules/dataverse-parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 7899c74bb44..9be364efdc0 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -199,7 +199,7 @@ 1.7.0 - 0.45.0 + 0.46.0 From c7b16bd359eb77de182ef6e34bb9ec121e88df0f Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 00:27:43 +0200 Subject: [PATCH 54/66] style(ct): remove duplicate heading in ConfigBaker README Eliminated a redundant "Supported Image Tags" heading for improved clarity and readability in the documentation. --- modules/container-configbaker/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/container-configbaker/README.md b/modules/container-configbaker/README.md index 38118f18b6d..75862ee0809 100644 --- a/modules/container-configbaker/README.md +++ b/modules/container-configbaker/README.md @@ -24,8 +24,6 @@ to ask for help and guidance. ## Supported Image Tags -## Supported Image Tags - This image is sourced within the main upstream code [repository of the Dataverse software](https://github.com/IQSS/dataverse). Development and maintenance of the [image's code](https://github.com/IQSS/dataverse/tree/develop/modules/container-configbaker) happens there (again, by the community). From 7708149393bbf26c4e50db9607e59522b5a79779 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 00:59:00 +0200 Subject: [PATCH 55/66] docs(ct): add release notes for container image versioning improvements Summarized enhancements including versioned tags, workflow reorganization, backport support, base image migration to Ubuntu, and improved documentation for better lifecycle management and operational clarity. --- doc/release-notes/10618-app-image-tags.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/release-notes/10618-app-image-tags.md diff --git a/doc/release-notes/10618-app-image-tags.md b/doc/release-notes/10618-app-image-tags.md new file mode 100644 index 00000000000..3942146ea54 --- /dev/null +++ b/doc/release-notes/10618-app-image-tags.md @@ -0,0 +1,17 @@ +### Container Image Versioning and Maintenance Improvements + +Container image management has been enhanced to provide better support for multiple Dataverse releases and improved maintenance workflows. + +**Versioned Image Tags**: Application and Config Baker images now follow the Bitnami pattern with versioned tags; supporting the latest three Dataverse software releases. This enables users to pin to specific versions and especially provides better stability for production deployments. + +**Workflow Responsibility Split**: GitHub Actions workflows for containers have been reorganized with a clear separation of concerns: +- `container_maintenance.yml` handles all release-time and maintenance activities +- Other workflows focus solely on preview images for development merges and pull requests + +**Backport Support**: Application and Config Baker image builds now support including code backports for past releases, enabling the delivery of security fixes and critical updates to older (supported) versions. + +**Config Baker Base Image Change**: The Config Baker image has been migrated from Alpine to Ubuntu as its base operating system, aligning with other container images in the project for consistency and better compatibility. The past releases have not been migrated, only future releases (6.7+) will use Ubuntu. + +**Enhanced Documentation**: Container image documentation has been updated to reflect the new versioning scheme and maintenance processes. + +These improvements provide more robust container image lifecycle management, better security update delivery, and clearer operational procedures for both development and production environments. From b70b92b476f219bfee62ed32a2a66e19a750f8bb Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 01:10:44 +0200 Subject: [PATCH 56/66] chore(ct): update demo compose and docs to use `latest` image tags Switched references from `alpha` to `latest` tags in demo compose file and documentation for Dataverse and ConfigBaker images. Adjusted related explanations to reflect this change and provided guidance on using specific version tags for consistency. --- doc/sphinx-guides/source/container/running/demo.rst | 7 +++++-- doc/sphinx-guides/source/container/running/production.rst | 7 +------ docker/compose/demo/compose.yml | 8 ++++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index 162632acc68..658cbda363a 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -61,7 +61,7 @@ Edit the ``compose.yml`` file and look for the following section. bootstrap: container_name: "bootstrap" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" environment: - TIMEOUT=3m @@ -291,7 +291,10 @@ Additional containers are used in development (see :doc:`../dev-usage`), but for Tags and Versions +++++++++++++++++ -The compose file references a tag called "alpha", which corresponds to the latest released version of Dataverse. This means that if a release of Dataverse comes out while you are demo'ing or evaluating, the version of Dataverse you are using could change if you do a ``docker pull``. We are aware that there is a desire for tags that correspond to versions to ensure consistency. You are welcome to join `the discussion `_ and otherwise get in touch (see :ref:`helping-containers`). For more on tags, see :ref:`app-image-supported-tags`. +The compose file references a tag called "latest", which corresponds to the latest released version of Dataverse. +This means that if a release of Dataverse comes out while you are demo'ing or evaluating, the version of Dataverse you are using could change if you do a ``docker pull``. +Feel free to change it to a specific version to avoid this. +For more on available tags, see supported tags section for :ref:`Application ` and :ref:`Config Baker ` images. Once Dataverse is running, you can check which version you have through the normal methods: diff --git a/doc/sphinx-guides/source/container/running/production.rst b/doc/sphinx-guides/source/container/running/production.rst index 3294db8ec1b..1c12798509e 100644 --- a/doc/sphinx-guides/source/container/running/production.rst +++ b/doc/sphinx-guides/source/container/running/production.rst @@ -9,12 +9,7 @@ Status The images described in this guide are not yet recommended for production usage, but we think we are close. We'd like to make the following improvements: -- Tagged releases - - - Currently, you have the choice between "alpha" images that change under your feet every time a new version of Dataverse is released or "unstable" images that track the "develop" branch, which is updated frequently. Instead, we'd like to offer images like 6.4, 6.5, etc. We are tracking this work at https://github.com/IQSS/dataverse/issues/10478 and there is some preliminary code at https://github.com/IQSS/dataverse/tree/10478-version-base-img . You are welcome to join the following discussions: - - - https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/change.20version.20scheme.20base.20image.3F/near/405636949 - - https://dataverse.zulipchat.com/#narrow/stream/375812-containers/topic/tagging.20images.20with.20versions/near/366600747 +- Tagged releases (done, see supported tags section for :ref:`Application ` and :ref:`Config Baker ` images) - More docs on setting up additional features diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index ed421cc49b7..5f3ce59055c 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -5,7 +5,7 @@ services: dataverse: container_name: "dataverse" hostname: dataverse - image: gdcc/dataverse:alpha + image: gdcc/dataverse:latest restart: on-failure user: payara environment: @@ -50,7 +50,7 @@ services: bootstrap: container_name: "bootstrap" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" environment: - TIMEOUT=3m @@ -65,7 +65,7 @@ services: dv_initializer: container_name: "dv_initializer" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" command: - sh @@ -122,7 +122,7 @@ services: solr_initializer: container_name: "solr_initializer" - image: gdcc/configbaker:alpha + image: gdcc/configbaker:latest restart: "no" command: - sh From 5bbfb05657c643f001aa7f29d01363ce112a6624 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 01:20:49 +0200 Subject: [PATCH 57/66] ci(ct): add explicit read permissions for contents and packages in image workflows Ensured required permissions for `contents` and `packages` in `application-image` and `configbaker-image` workflows to align with best practices and enhance security. --- .github/workflows/container_maintenance.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index 722e1f5fe71..d42ea41e55a 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -102,6 +102,9 @@ jobs: application-image: name: "Application Image Matrix Build" runs-on: ubuntu-latest + permissions: + contents: read + packages: read needs: - discover - base-image @@ -144,6 +147,9 @@ jobs: configbaker-image: name: "ConfigBaker Image Matrix Build" runs-on: ubuntu-latest + permissions: + contents: read + packages: read needs: - discover # Only run in upstream repo - avoid unnecessary runs in forks. From 4de642efff995d49067756c5d04192f1439be617 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Jun 2025 14:03:35 -0400 Subject: [PATCH 58/66] let doc writers know to pip install new requirements #10618 --- doc/release-notes/10618-app-image-tags.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release-notes/10618-app-image-tags.md b/doc/release-notes/10618-app-image-tags.md index 3942146ea54..dbbcc9e8ea4 100644 --- a/doc/release-notes/10618-app-image-tags.md +++ b/doc/release-notes/10618-app-image-tags.md @@ -15,3 +15,7 @@ Container image management has been enhanced to provide better support for multi **Enhanced Documentation**: Container image documentation has been updated to reflect the new versioning scheme and maintenance processes. These improvements provide more robust container image lifecycle management, better security update delivery, and clearer operational procedures for both development and production environments. + +## Notes for documentation writers + +Sphinx has been upgraded to 7.4.0 and new dependencies been added, including semver. Please re-run the `pip install -r requirements.txt` setup [step](https://guides.dataverse.org/en/6.7/contributor/documentation.html#installing-sphinx) to upgrade your environment. Otherwise you might see an error like `ModuleNotFoundError: No module named 'semver'`. From ae03e73d2989308ab94e439564b1cef9e8613d44 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Jun 2025 14:19:27 -0400 Subject: [PATCH 59/66] improve release note #10618 --- doc/release-notes/10618-app-image-tags.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/doc/release-notes/10618-app-image-tags.md b/doc/release-notes/10618-app-image-tags.md index dbbcc9e8ea4..9eeb723998a 100644 --- a/doc/release-notes/10618-app-image-tags.md +++ b/doc/release-notes/10618-app-image-tags.md @@ -1,20 +1,22 @@ -### Container Image Versioning and Maintenance Improvements +### Ability to pin to a specific Dataverse version in Docker (and more) Container image management has been enhanced to provide better support for multiple Dataverse releases and improved maintenance workflows. -**Versioned Image Tags**: Application and Config Baker images now follow the Bitnami pattern with versioned tags; supporting the latest three Dataverse software releases. This enables users to pin to specific versions and especially provides better stability for production deployments. - -**Workflow Responsibility Split**: GitHub Actions workflows for containers have been reorganized with a clear separation of concerns: -- `container_maintenance.yml` handles all release-time and maintenance activities -- Other workflows focus solely on preview images for development merges and pull requests +**Versioned Image Tags**: Application ("dataverse") and Config Baker [images on Docker Hub](https://hub.docker.com/u/gdcc) now have versioned tags, supporting the latest three Dataverse software releases. This enables users to pin to specific versions (e.g. 6.7), providing better stability for production deployments. Previously, the "alpha" tag could be used, but it was always overwritten by the latest release. Now, you can choose the 6.7 tag, for example, to stay on that version. **Backport Support**: Application and Config Baker image builds now support including code backports for past releases, enabling the delivery of security fixes and critical updates to older (supported) versions. +**Enhanced Documentation**: Container image [documentation](https://dataverse-guide--11477.org.readthedocs.build/en/11477/container/index.html) has been updated to reflect the new versioning scheme and maintenance processes. + **Config Baker Base Image Change**: The Config Baker image has been migrated from Alpine to Ubuntu as its base operating system, aligning with other container images in the project for consistency and better compatibility. The past releases have not been migrated, only future releases (6.7+) will use Ubuntu. -**Enhanced Documentation**: Container image documentation has been updated to reflect the new versioning scheme and maintenance processes. +**Workflow Responsibility Split**: GitHub Actions workflows for containers have been reorganized with a clear separation of concerns: + +- `container_maintenance.yml` handles all release-time and maintenance activities +- Other workflows focus solely on preview images for development merges and pull requests These improvements provide more robust container image lifecycle management, better security update delivery, and clearer operational procedures for both development and production environments. +See also the [Container Guide](https://dataverse-guide--11477.org.readthedocs.build/en/11477/container/index.html), #10618, and #11477. ## Notes for documentation writers From df7c196bb5cc76d0a3023b245036b2dd7904dd9d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Jun 2025 14:40:49 -0400 Subject: [PATCH 60/66] make use of "version" string in "making releases" doc #10618 --- doc/sphinx-guides/source/developers/making-releases.rst | 6 +++--- doc/sphinx-guides/source/versions.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/sphinx-guides/source/developers/making-releases.rst b/doc/sphinx-guides/source/developers/making-releases.rst index 14ad87ad600..01bd729a62f 100755 --- a/doc/sphinx-guides/source/developers/making-releases.rst +++ b/doc/sphinx-guides/source/developers/making-releases.rst @@ -135,11 +135,11 @@ Make the following changes in the release branch. Increment the version number to the milestone (e.g. 5.10.1) in the following two files: - modules/dataverse-parent/pom.xml -> ```` -> ```` (e.g. `pom.xml commit `_) -- doc/sphinx-guides/source/conf.py (two places, e.g. `conf.py commit `_) +- doc/sphinx-guides/source/conf.py -Add the version being released to the lists in the following file: +In the following ``versions.rst`` file: -- doc/sphinx-guides/source/versions.rst (e.g. `versions.rst commit `_) +- doc/sphinx-guides/source/versions.rst - Below the ``- |version|`` bullet (``|version|`` comes from the ``conf.py`` file you just edited), add a bullet for what is soon to be the previous release. Return to the parent pom and make the following change, which is necessary for proper tagging of images: diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index cd19837dff1..e877efde83a 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,7 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 6.6 +- |version| - `6.5 `__ - `6.4 `__ - `6.3 `__ From 20c8696d06a1b6be3ca544a9cddbbc67bb98180d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Jun 2025 15:37:53 -0400 Subject: [PATCH 61/66] various tweaks to docs #10618 --- doc/sphinx-guides/source/container/app-image.rst | 11 ++++++----- doc/sphinx-guides/source/container/base-image.rst | 2 +- .../source/container/configbaker-image.rst | 10 +++++----- doc/sphinx-guides/source/container/running/demo.rst | 2 ++ .../source/container/running/production.rst | 5 ++--- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index df04423f26d..0b3b472a147 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -31,7 +31,7 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). -All supported images are signed up for scheduled maintenance, executed every Sunday. +All supported images receive scheduled maintenance, executed every Sunday. New revisions are kept to a minimum, usually created when some dependency needs (security) updates. For the application images it correlates mostly to the :doc:`base image ` receiving updates. @@ -47,7 +47,7 @@ You can find all the tags at https://hub.docker.com/r/gdcc/dataverse/tags Tags for Production Use ^^^^^^^^^^^^^^^^^^^^^^^ -The images of the three latest releases of the Dataverse project will receive updates such as security patches for the underlying operating system. +The images of the three latest releases of Dataverse will receive updates such as security patches for the underlying operating system. Content will be fairly stable as disruptive changes like Payara or Java upgrades will be handled in a new major or minor upgrade to Dataverse (a new ``.`` tag). Expect disruptive changes in case of high risk security threats. @@ -67,9 +67,9 @@ Expect disruptive changes in case of high risk security threats. Tags for Development Use ^^^^^^^^^^^^^^^^^^^^^^^^ -All of the tags below are strongly recommended for development purposes only due to their fast changing nature. -In addition to updates due to PR merges, the most recent are undergoing scheduled maintenance to ensure timely security fixes. -When a development cycle of the Dataverse project finishes, maintenance ceases for any tags carrying version numbers. +All of the tags below are strongly recommended only for development purposes due to their fast-changing nature. +In addition to updates due to PR merges, the most recent tags undergo scheduled maintenance to ensure timely security fixes. +When a development cycle of Dataverse finishes (see :doc:`/developers/making-releases`), maintenance ceases for any of the tags below carrying version numbers. For now, stale images will be kept on Docker Hub indefinitely. - | **Unstable** @@ -81,6 +81,7 @@ For now, stale images will be kept on Docker Hub indefinitely. | Example: :substitution-code:`|nextVersion|-noble` | Summary: Rolling tag, equivalent to ``unstable`` for current development cycle. Will roll over to the rolling production tag after a Dataverse release. + | Discussion: Perhaps you are eager to starting testing features of an upcoming version (e.g. |nextVersion|) in a staging environment. You select the :substitution-code:`|nextVersion|-noble` tag (as opposed to ``unstable``) because you want to stay on |nextVersion| rather switching to the version **after that** when a release is made (which would happen if you had selected the ``unstable`` tag). Also, when the next release comes out (|nextVersion| in this example), you would stay on the :substitution-code:`|nextVersion|-noble` tag, which is the same tag that someone would use who wants the final release of |nextVersion|. (See "Rolling Production", above.) **NOTE**: In these tags for development usage, the version number will always be 1 minor version ahead of existing Dataverse releases. Example: Assume Dataverse ``6.x`` is released, ``6.(x+1)`` is underway. diff --git a/doc/sphinx-guides/source/container/base-image.rst b/doc/sphinx-guides/source/container/base-image.rst index 86deebdaa1f..d279135dac4 100644 --- a/doc/sphinx-guides/source/container/base-image.rst +++ b/doc/sphinx-guides/source/container/base-image.rst @@ -30,7 +30,7 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). -All supported images are signed up for scheduled maintenance, executed every Sunday. +All supported images receive scheduled maintenance, executed every Sunday. New revisions are kept to a minimum, usually created when some dependency needs (security) updates. (Examples: JRE patch releases, ImageMagick fixes, etc.) diff --git a/doc/sphinx-guides/source/container/configbaker-image.rst b/doc/sphinx-guides/source/container/configbaker-image.rst index b303594ea8f..4cc64fb150e 100644 --- a/doc/sphinx-guides/source/container/configbaker-image.rst +++ b/doc/sphinx-guides/source/container/configbaker-image.rst @@ -23,9 +23,9 @@ This image is sourced from the main upstream code `repository of the Dataverse s Development and maintenance of the `image's code `_ happens there (again, by the community). -All supported images are signed up for scheduled maintenance, executed every Sunday. +All supported images receive scheduled maintenance, executed every Sunday. New revisions are kept to a minimum, usually created when some dependency needs (security) updates. -Be advised: the Trivy Scanner is used to check for fixed vulnerabilities and rebuilds are issued if such a fix is detected. +The `Trivy `_ scanner is used to check for fixed vulnerabilities and rebuilds are issued if such a fix is detected. Our tagging is inspired by `Bitnami `_ and we offer two categories of tags: @@ -59,9 +59,9 @@ Expect disruptive changes in case of high risk security threats. Tags for Development Use ^^^^^^^^^^^^^^^^^^^^^^^^ -All of the tags below are strongly recommended for development purposes only due to their fast changing nature. -In addition to updates due to PR merges, the most recent are undergoing scheduled maintenance to ensure timely security fixes. -When a development cycle of the Dataverse project finishes, maintenance ceases for any tags carrying version numbers. +All of the tags below are strongly recommended only for development purposes due to their fast-changing nature. +In addition to updates due to PR merges, the most recent tags undergo scheduled maintenance to ensure timely security fixes. +When a development cycle of Dataverse finishes, maintenance ceases for any tags carrying version numbers. For now, stale images will be kept on Docker Hub indefinitely. - | **Unstable** diff --git a/doc/sphinx-guides/source/container/running/demo.rst b/doc/sphinx-guides/source/container/running/demo.rst index 658cbda363a..d4afee8a18a 100644 --- a/doc/sphinx-guides/source/container/running/demo.rst +++ b/doc/sphinx-guides/source/container/running/demo.rst @@ -192,6 +192,8 @@ PID Providers Dataverse supports multiple Persistent ID (PID) providers. The ``compose.yml`` file uses the Permalink PID provider. Follow :ref:`pids-configuration` to reconfigure as needed. +.. _file-previewers-ct: + File Previewers +++++++++++++++ diff --git a/doc/sphinx-guides/source/container/running/production.rst b/doc/sphinx-guides/source/container/running/production.rst index 1c12798509e..4fe16447d7e 100644 --- a/doc/sphinx-guides/source/container/running/production.rst +++ b/doc/sphinx-guides/source/container/running/production.rst @@ -7,13 +7,12 @@ Production (Future) Status ------ -The images described in this guide are not yet recommended for production usage, but we think we are close. We'd like to make the following improvements: +The images described in this guide are not yet recommended for production usage, but we think we are close. (Tagged releases are done; see the "supported image tags" section for :ref:`Application ` and :ref:`Config Baker ` images.) For now, please see :doc:`demo`. -- Tagged releases (done, see supported tags section for :ref:`Application ` and :ref:`Config Baker ` images) +We'd like to make the following improvements: - More docs on setting up additional features - - How to set up previewers. See https://github.com/IQSS/dataverse/issues/10506 - How to set up Rserve. - Go through all the features in docs and check what needs to be done differently with containers From 968e89e86dd149446c885d177be557b4120043fa Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Jun 2025 15:39:07 -0400 Subject: [PATCH 62/66] typo #10618 --- doc/release-notes/10618-app-image-tags.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10618-app-image-tags.md b/doc/release-notes/10618-app-image-tags.md index 9eeb723998a..fc8e827efdb 100644 --- a/doc/release-notes/10618-app-image-tags.md +++ b/doc/release-notes/10618-app-image-tags.md @@ -20,4 +20,4 @@ See also the [Container Guide](https://dataverse-guide--11477.org.readthedocs.bu ## Notes for documentation writers -Sphinx has been upgraded to 7.4.0 and new dependencies been added, including semver. Please re-run the `pip install -r requirements.txt` setup [step](https://guides.dataverse.org/en/6.7/contributor/documentation.html#installing-sphinx) to upgrade your environment. Otherwise you might see an error like `ModuleNotFoundError: No module named 'semver'`. +Sphinx has been upgraded to 7.4.0 and new dependencies have been added, including semver. Please re-run the `pip install -r requirements.txt` setup [step](https://guides.dataverse.org/en/6.7/contributor/documentation.html#installing-sphinx) to upgrade your environment. Otherwise you might see an error like `ModuleNotFoundError: No module named 'semver'`. From 8b971a8fb1185546ebbdc0d344b1312195890cfa Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Jun 2025 15:58:02 -0400 Subject: [PATCH 63/66] mention alpha tag is going away #10618 --- doc/release-notes/10618-app-image-tags.md | 2 +- doc/sphinx-guides/source/container/app-image.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/10618-app-image-tags.md b/doc/release-notes/10618-app-image-tags.md index fc8e827efdb..84d73dc036e 100644 --- a/doc/release-notes/10618-app-image-tags.md +++ b/doc/release-notes/10618-app-image-tags.md @@ -2,7 +2,7 @@ Container image management has been enhanced to provide better support for multiple Dataverse releases and improved maintenance workflows. -**Versioned Image Tags**: Application ("dataverse") and Config Baker [images on Docker Hub](https://hub.docker.com/u/gdcc) now have versioned tags, supporting the latest three Dataverse software releases. This enables users to pin to specific versions (e.g. 6.7), providing better stability for production deployments. Previously, the "alpha" tag could be used, but it was always overwritten by the latest release. Now, you can choose the 6.7 tag, for example, to stay on that version. +**Versioned Image Tags**: Application ("dataverse") and Config Baker [images on Docker Hub](https://hub.docker.com/u/gdcc) now have versioned tags, supporting the latest three Dataverse software releases. This enables users to pin to specific versions (e.g. 6.7), providing better stability for production deployments. Previously, the "alpha" tag could be used, but it was always overwritten by the latest release. Now, you can choose the 6.7 tag, for example, to stay on that version. Please note that the "alpha" tag should no longer be used and will likely be deleted. The equivalent is the new "latest" tag. **Backport Support**: Application and Config Baker image builds now support including code backports for past releases, enabling the delivery of security fixes and critical updates to older (supported) versions. diff --git a/doc/sphinx-guides/source/container/app-image.rst b/doc/sphinx-guides/source/container/app-image.rst index 0b3b472a147..afffeae1c0b 100644 --- a/doc/sphinx-guides/source/container/app-image.rst +++ b/doc/sphinx-guides/source/container/app-image.rst @@ -53,7 +53,7 @@ Expect disruptive changes in case of high risk security threats. - | **Latest** | Definition: ``latest`` - | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. + | Summary: Rolling tag, always pointing to the latest revision of the most current Dataverse release. In Dataverse 6.6 and lower, the equivalent was the ``alpha`` tag, which has been deleted. - | **Rolling Production** | Definition: ``.-`` | Example: :substitution-code:`|version|-noble` From 095b70e3d8163a725ee8ad3d6ba947bf1787dfa3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 4 Jun 2025 16:11:11 -0400 Subject: [PATCH 64/66] clarify --- .github/workflows/maven_unit_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 44c6117dd9c..efefdaee02a 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -2,8 +2,8 @@ name: Maven Tests on: push: - # Only run for development and feature branches but not tags or the stable branch. - # For tags (=released) it would be too late anyway, and there is always a PR for the master branch. + # Only run for development and feature branches. Don't waste CPU cycles testing + # master when the PR to update it from develop already ran these tests. branches: - '*' - '!master' From 388c55fbc465d3955b4cfbce0ee7553754b124b1 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Wed, 4 Jun 2025 23:54:52 +0200 Subject: [PATCH 65/66] ci(ct): re-enable conditional workflows in upstream repository Removed temporary testing condition for `gdcc/wip-base-image` and reinstated the `if` condition to restrict workflow runs to the upstream `IQSS` repository. --- .github/workflows/container_maintenance.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/container_maintenance.yml b/.github/workflows/container_maintenance.yml index d42ea41e55a..8abe33bdefc 100644 --- a/.github/workflows/container_maintenance.yml +++ b/.github/workflows/container_maintenance.yml @@ -58,8 +58,7 @@ jobs: contents: read packages: read # Only run in upstream repo - avoid unnecessary runs in forks - # TODO: re-enable once we are done testing in gdcc/wip-base-image project - # if: ${{ github.repository_owner == 'IQSS' }} + if: ${{ github.repository_owner == 'IQSS' }} needs: - discover outputs: @@ -109,10 +108,8 @@ jobs: - discover - base-image # Only run in upstream repo - avoid unnecessary runs in forks. - # TODO: If we add a push trigger later, we might want to prepend "always() &&" to ignore the status of the base job. - # Needs further investigation. - # TODO: re-enable once we are done testing in gdcc/wip-base-image project - # if: ${{ github.repository_owner == 'IQSS' }} + # TODO: If we add a push trigger later, we might want to prepend "always() &&" to ignore the status of the base job. Needs further investigation. + if: ${{ github.repository_owner == 'IQSS' }} outputs: supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} @@ -153,10 +150,8 @@ jobs: needs: - discover # Only run in upstream repo - avoid unnecessary runs in forks. - # TODO: If we add a push trigger later, we might want to prepend "always() &&" to ignore the status of the base job. - # Needs further investigation. - # TODO: re-enable once we are done testing in gdcc/wip-base-image project - # if: ${{ github.repository_owner == 'IQSS' }} + # TODO: If we add a push trigger later, we might want to prepend "always() &&" to ignore the status of the base job. Needs further investigation. + if: ${{ github.repository_owner == 'IQSS' }} outputs: supported_tag_matrix: ${{ steps.execute.outputs.supported_tag_matrix }} rebuilt_images: ${{ steps.execute.outputs.rebuilt_images }} From fafd87150435e2ac1655fa504debcff8c84f62b9 Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Thu, 5 Jun 2025 00:33:09 +0200 Subject: [PATCH 66/66] docs(ct): add README for backports directory and usage guidelines Introduced documentation detailing the structure, purpose, and usage of the `src/backports` directory to support consistent patch management across multiple Dataverse release versions. --- src/backports/README.md | 81 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/backports/README.md diff --git a/src/backports/README.md b/src/backports/README.md new file mode 100644 index 00000000000..420e1dbd466 --- /dev/null +++ b/src/backports/README.md @@ -0,0 +1,81 @@ +# Backports Directory + +This directory contains patch files for backporting changes across different Dataverse releases. +These changes can be features, build enhancements, security patches, and more. + +## Directory Structure + +The backports directory is organized by "release version" (or "release tag") folders. +Each folder name corresponds to a [Dataverse release](https://github.com/IQSS/dataverse/releases) and [its associated Git tag](https://github.com/IQSS/dataverse/tags): + +``` +src/backports/ +├── v6.4/ # Patches for Dataverse 6.4 +├── v6.5/ # Patches for Dataverse 6.5 +├── v6.6/ # Patches for Dataverse 6.6 +└── ... +``` + +Each version folder contains numbered patch files that modify specific components. + +For example: +- `001-parent-pom.xml.patch` -> Modifications to the parent POM configuration +- `002-pom.xml.patch` -> Changes to the main POM file +- Additional patches as needed. + +## Intended Usage + +The patch files in each release folder are designed to be applied in numerical order. + +Currently, they are primarily used by the container maintenance workflows (see `.github/workflows/container_maintenance.yml`) to ensure maintenance script compatibility across different Dataverse versions. +See also the "three releases back" support promise at https://guides.dataverse.org/en/latest/container/app-image.html#tags-for-production-use. + +*Note: The backport patches mechanism is by no means limited to usage in a container context.* +*They can be applied manually or from some other automated release process to backports changes to older releases.* + +## Creating Patch Files + +To create a new patch file using `git diff`, follow these steps: + +### Example: Creating a POM patch + +1. **Make your changes** to the target file(s) on the appropriate branch (usually your feature branch). +2. **Head to the root** of the Git repository. +3. **Generate the patch** using `git diff`: + ```bash + # Create a patch for changes to pom.xml, comparing with the pom.xml contained in the v6.5 tag: + git --no-pager diff v6.5 pom.xml > src/backports/v6.5/002-pom.xml.patch + + # Or create a patch for multiple files: + git --no-pager diff v6.5 modules/dataverse-parent/pom.xml pom.xml > src/backports/v6.5/003-multi-pom.patch + + # Create a patch from staged changes + git diff --cached > src/backports/v6.5/004-staged-changes.patch + ``` + +4. **Review the patch** to ensure it contains only the intended changes: + ```bash + cat src/backports/v6.5/002-pom.xml.patch + ``` + +5. **Repeat for other tags** as necessary. + +### Patch Naming Convention + +Use the following naming pattern: +- `001-` prefix with three-digit numbering for ordering +- Descriptive name indicating what is being patched +- `.patch` file extension + +Examples: +- `001-parent-pom.xml.patch` +- `002-pom.xml.patch` +- `003-dockerfile-updates.patch` + +## Integration with CI/CD + +These patches are automatically applied during the container maintenance workflows to ensure that older release versions can be built with updated dependencies and configurations while maintaining compatibility. +The patches support the multi-version container image strategy that builds and maintains Docker images for the current development branch plus the last three released versions. +See also the "three releases back" support promise at https://guides.dataverse.org/en/latest/container/app-image.html#tags-for-production-use. + +In the future, other automations may pick up the patches to release updated WAR files or similar. \ No newline at end of file