From b9663b8ac7abf785443a39f38914f64c65092133 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 8 Oct 2025 10:49:26 +0200 Subject: [PATCH 1/9] ci(build): Add runners selection --- .github/workflows/reusable_build_image.yaml | 52 +++++++++++++++++---- .github/workflows/runners/mixed.json | 10 ++++ .github/workflows/runners/ubicloud.json | 10 ++++ 3 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/runners/mixed.json create mode 100644 .github/workflows/runners/ubicloud.json diff --git a/.github/workflows/reusable_build_image.yaml b/.github/workflows/reusable_build_image.yaml index ceeb91132..0f4011c43 100644 --- a/.github/workflows/reusable_build_image.yaml +++ b/.github/workflows/reusable_build_image.yaml @@ -10,6 +10,13 @@ on: registry-namespace: required: true type: string + runners: + description: | + The type of runners used. + - `mixed`: x86_64 builds run on the GitHub hosted runners and the aarch64 builds run on the Ubicloud runners + - `ubicloud`: Both the x86_64 and the aarch64 builds run on the Ubicloud runners + default: mixed + type: string secrets: harbor-robot-secret: description: The secret for the Harbor robot user used to push images and manifest @@ -19,8 +26,33 @@ on: required: true jobs: - generate_matrix: - name: Generate Version List + generate_runner_dimension: + name: Generate Runner Dimension + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - id: runners + shell: bash + env: + RUNNERS_TYPE: ${{ inputs.runners }} + run: | + if [ "$RUNNERS_TYPE" == "mixed" ]; then + RUNNERS=$(cat .github/workflows/runners/mixed.json) + echo "RUNNERS< 1 && !cancelled()) steps: diff --git a/.github/workflows/runners/mixed.json b/.github/workflows/runners/mixed.json new file mode 100644 index 000000000..772ea676e --- /dev/null +++ b/.github/workflows/runners/mixed.json @@ -0,0 +1,10 @@ +[ + { + "name": "ubuntu-latest", + "arch": "amd64" + }, + { + "name": "ubicloud-standard-8-arm-ubuntu-2404", + "arch": "arm64" + } +] diff --git a/.github/workflows/runners/ubicloud.json b/.github/workflows/runners/ubicloud.json new file mode 100644 index 000000000..a871cbbcf --- /dev/null +++ b/.github/workflows/runners/ubicloud.json @@ -0,0 +1,10 @@ +[ + { + "name": "ubicloud-standard-8-ubuntu-2404", + "arch": "amd64" + }, + { + "name": "ubicloud-standard-8-arm-ubuntu-2404", + "arch": "arm64" + } +] From fc8f28efaa566e6206942c2b1e3b8ac5ca1b826f Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 8 Oct 2025 11:03:12 +0200 Subject: [PATCH 2/9] ci(build): Use jq to produce JSOn without newlines --- .github/workflows/reusable_build_image.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/reusable_build_image.yaml b/.github/workflows/reusable_build_image.yaml index 0f4011c43..94b21843a 100644 --- a/.github/workflows/reusable_build_image.yaml +++ b/.github/workflows/reusable_build_image.yaml @@ -39,11 +39,11 @@ jobs: RUNNERS_TYPE: ${{ inputs.runners }} run: | if [ "$RUNNERS_TYPE" == "mixed" ]; then - RUNNERS=$(cat .github/workflows/runners/mixed.json) - echo "RUNNERS< Date: Wed, 8 Oct 2025 11:08:03 +0200 Subject: [PATCH 3/9] ci(build): Remove superfluous cat from command --- .github/workflows/reusable_build_image.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable_build_image.yaml b/.github/workflows/reusable_build_image.yaml index 94b21843a..e5dad443a 100644 --- a/.github/workflows/reusable_build_image.yaml +++ b/.github/workflows/reusable_build_image.yaml @@ -39,10 +39,10 @@ jobs: RUNNERS_TYPE: ${{ inputs.runners }} run: | if [ "$RUNNERS_TYPE" == "mixed" ]; then - RUNNERS=$(cat .github/workflows/runners/mixed.json | jq --compact-output) + RUNNERS=$(jq --compact-output < .github/workflows/runners/mixed.json) echo "RUNNERS=$RUNNERS" | tee -a "$GITHUB_OUTPUT" elif [ "$RUNNERS_TYPE" == "ubicloud" ]; then - RUNNERS=$(cat .github/workflows/runners/ubicloud.json | jq --compact-output) + RUNNERS=$(jq --compact-output < .github/workflows/runners/ubicloud.json) echo "RUNNERS=$RUNNERS" | tee -a "$GITHUB_OUTPUT" else echo "Invalid runners type, expected 'mixed' or 'ubicloud'" From 15ef0fb963f6202a1360b0fa44285d8cd5247651 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 8 Oct 2025 11:11:41 +0200 Subject: [PATCH 4/9] temp(ci/opa): Use only Ubicloud runners This ensures that the build succeeds with the set of Ubicloud runners specified in the JSON file. --- .github/workflows/build_opa.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_opa.yaml b/.github/workflows/build_opa.yaml index 54400f5fa..8d14cd5bd 100644 --- a/.github/workflows/build_opa.yaml +++ b/.github/workflows/build_opa.yaml @@ -6,7 +6,7 @@ run-name: | on: workflow_dispatch: schedule: - - cron: '0 2 1/2 * *' # https://crontab.guru/#0_2_1/2_*_* + - cron: "0 2 1/2 * *" # https://crontab.guru/#0_2_1/2_*_* push: branches: [main] tags: @@ -33,3 +33,4 @@ jobs: product-name: opa sdp-version: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} registry-namespace: sdp + runners: ubicloud From 2b9ccc45f6b7774464cca38fc74649478c11007f Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 8 Oct 2025 11:26:28 +0200 Subject: [PATCH 5/9] Revert "temp(ci/opa): Use only Ubicloud runners" This reverts commit 15ef0fb963f6202a1360b0fa44285d8cd5247651. --- .github/workflows/build_opa.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_opa.yaml b/.github/workflows/build_opa.yaml index 8d14cd5bd..54400f5fa 100644 --- a/.github/workflows/build_opa.yaml +++ b/.github/workflows/build_opa.yaml @@ -6,7 +6,7 @@ run-name: | on: workflow_dispatch: schedule: - - cron: "0 2 1/2 * *" # https://crontab.guru/#0_2_1/2_*_* + - cron: '0 2 1/2 * *' # https://crontab.guru/#0_2_1/2_*_* push: branches: [main] tags: @@ -33,4 +33,3 @@ jobs: product-name: opa sdp-version: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} registry-namespace: sdp - runners: ubicloud From 692d34df0533a151ffa3aada6a663d5ef6354eba Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 8 Oct 2025 11:27:17 +0200 Subject: [PATCH 6/9] ci(opensearch-dashboards): Use Ubicloud runners --- .github/workflows/build_opensearch_dashboards.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_opensearch_dashboards.yaml b/.github/workflows/build_opensearch_dashboards.yaml index 392b6f340..7f014937d 100644 --- a/.github/workflows/build_opensearch_dashboards.yaml +++ b/.github/workflows/build_opensearch_dashboards.yaml @@ -6,7 +6,7 @@ run-name: | on: workflow_dispatch: schedule: - - cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_* + - cron: "0 0 2/2 * *" # https://crontab.guru/#0_0_2/2_*_* push: branches: [main] tags: @@ -34,3 +34,4 @@ jobs: product-name: opensearch-dashboards sdp-version: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} registry-namespace: sdp + runners: ubicloud From 54cd9b639b1eeef21b240960df2cac936cf1dcbe Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 8 Oct 2025 11:39:41 +0200 Subject: [PATCH 7/9] chore: Update actionlint config file --- .github/actionlint.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 8337548a1..60dce63b8 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,4 +2,5 @@ self-hosted-runner: # Ubicloud machines we are using labels: - - ubicloud-standard-8-arm + - ubicloud-standard-8-arm-ubuntu-2404 + - ubicloud-standard-8-ubuntu-2404 From de22cd10c8e118dec7234b9ed9bed05b4ed2af28 Mon Sep 17 00:00:00 2001 From: Techassi Date: Wed, 8 Oct 2025 11:40:03 +0200 Subject: [PATCH 8/9] chore: Update badges in README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c94304772..0846d41bd 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository contains Dockerfiles and scripts to build base images for use wi | [![Build Airflow]][build_airflow.yaml] | [![Build Druid]][build_druid.yaml] | [![Build Hadoop]][build_hadoop.yaml] | [![Build HBase]][build_hbase.yaml] | | [![Build Hive]][build_hive.yaml] | [![Build Java Base]][build_java-base.yaml] | [![Build Java Development]][build_java-devel.yaml] | [![Build Kafka Testing Tools]][build_kafka-testing-tools.yaml] | | [![Build Kafka]][build_kafka.yaml] | [![Build Krb5]][build_krb5.yaml] | [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | -| [![Build OPA]][build_opa.yaml] | [![Build OpenSearch]][build_opensearch.yaml] | [![Build OpenSearch Dashboards]][build_opensearch_dashboards.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] | +| [![Build OPA]][build_opa.yaml] | [![Build OpenSearch Dashboards]][build_opensearch_dashboards.yaml] | [![Build OpenSearch]][build_opensearch.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] | | [![Build Spark K8s]][build_spark-k8s.yaml] | [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | | [![Build Tools]][build_tools.yaml] | [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | | [![Build ZooKeeper]][build_zookeeper.yaml] | | | | @@ -78,10 +78,10 @@ preflight check container oci.stackable.tech/sdp/:-stackable Date: Wed, 8 Oct 2025 11:47:11 +0200 Subject: [PATCH 9/9] Revert "chore: Update badges in README" This reverts commit de22cd10c8e118dec7234b9ed9bed05b4ed2af28. This needs to be reverted, because it doesn't pass CI even though it passes locally for me. With this reverted, it will fail locally again. This is very curious, because any differences in file sorting sue to different shells should be accounted for by the bash shebang in the script. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0846d41bd..c94304772 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This repository contains Dockerfiles and scripts to build base images for use wi | [![Build Airflow]][build_airflow.yaml] | [![Build Druid]][build_druid.yaml] | [![Build Hadoop]][build_hadoop.yaml] | [![Build HBase]][build_hbase.yaml] | | [![Build Hive]][build_hive.yaml] | [![Build Java Base]][build_java-base.yaml] | [![Build Java Development]][build_java-devel.yaml] | [![Build Kafka Testing Tools]][build_kafka-testing-tools.yaml] | | [![Build Kafka]][build_kafka.yaml] | [![Build Krb5]][build_krb5.yaml] | [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | -| [![Build OPA]][build_opa.yaml] | [![Build OpenSearch Dashboards]][build_opensearch_dashboards.yaml] | [![Build OpenSearch]][build_opensearch.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] | +| [![Build OPA]][build_opa.yaml] | [![Build OpenSearch]][build_opensearch.yaml] | [![Build OpenSearch Dashboards]][build_opensearch_dashboards.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] | | [![Build Spark K8s]][build_spark-k8s.yaml] | [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | | [![Build Tools]][build_tools.yaml] | [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | | [![Build ZooKeeper]][build_zookeeper.yaml] | | | | @@ -78,10 +78,10 @@ preflight check container oci.stackable.tech/sdp/:-stackable