From affae2377dce1722c8df3d5ea7450a37c71da333 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Fri, 8 Aug 2025 13:20:18 -0700 Subject: [PATCH 1/6] Replace buildkite jdk version check w/GH action Previously a weekly buildkite job would check if there is a newer JDK available than the one vendored in logstash. This job would fail until someone manually updated the JDK version in the `versions.yml` file in logstash. This commit reduces manual intervention by moving the responsibility to a GH action where there are patterns for creating PRs back to logstash. The action should run weekly and also be available in the actions UI. It is locked down with the same permissions as the version bump action. --- .../jdk_availability_check_pipeline.yml | 14 ----- .github/workflows/bump-java-version.yml | 2 +- build.gradle | 48 ++++++++------- catalog-info.yaml | 60 ------------------- ci/check_jdk_version_availability.sh | 2 +- 5 files changed, 28 insertions(+), 98 deletions(-) delete mode 100644 .buildkite/jdk_availability_check_pipeline.yml diff --git a/.buildkite/jdk_availability_check_pipeline.yml b/.buildkite/jdk_availability_check_pipeline.yml deleted file mode 100644 index 3fa826564e0..00000000000 --- a/.buildkite/jdk_availability_check_pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: - - label: "JDK Availability check" - key: "jdk-availability-check" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci" - cpu: "4" - memory: "6Gi" - ephemeralStorage: "100Gi" - command: | - set -euo pipefail - - source .buildkite/scripts/common/container-agent.sh - export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info" - ci/check_jdk_version_availability.sh \ No newline at end of file diff --git a/.github/workflows/bump-java-version.yml b/.github/workflows/bump-java-version.yml index ac66267e441..328870f9eee 100644 --- a/.github/workflows/bump-java-version.yml +++ b/.github/workflows/bump-java-version.yml @@ -16,4 +16,4 @@ jobs: - name: Stub step run: | echo "Stub to iterate via PR" - \ No newline at end of file + diff --git a/build.gradle b/build.gradle index b6987700c46..49e8e4b7803 100644 --- a/build.gradle +++ b/build.gradle @@ -42,6 +42,7 @@ apply from: "rubyUtils.gradle" import de.undercouch.gradle.tasks.download.Download import groovy.json.JsonSlurper +import java.util.regex.Pattern import org.logstash.gradle.tooling.ListProjectDependencies import org.logstash.gradle.tooling.ExtractBundledJdkVersion import org.logstash.gradle.tooling.SignAliasDefinitions @@ -748,23 +749,6 @@ class JDKDetails { return createElasticCatalogDownloadUrl() } - // throws an error iff local version in versions.yml doesn't match the latest from JVM catalog. - void checkLocalVersionMatchingLatest() { - // retrieve the metadata from remote - def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${major}_${osName}" - def catalogMetadataUrl = URI.create(url).toURL() - def catalogConnection = catalogMetadataUrl.openConnection() - catalogConnection.requestMethod = 'GET' - assert catalogConnection.responseCode == 200 - - def metadataRetrieved = catalogConnection.content.text - def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved) - - if (catalogMetadata.version != revision || catalogMetadata.revision != build) { - throw new GradleException("Found new jdk version. Please update version.yml to ${catalogMetadata.version} build ${catalogMetadata.revision}") - } - } - private String createElasticCatalogDownloadUrl() { // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK @@ -874,11 +858,31 @@ tasks.register("downloadJdk", Download) { } } -tasks.register("checkNewJdkVersion") { - // use Linux x86_64 as canary platform - def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, "linux", "x86_64") - // throws Gradle exception if local and remote doesn't match - jdkDetails.checkLocalVersionMatchingLatest() +tasks.register("updateJdkVersion") { + description = "Update JDK version in versions.yml to match latest from JVM catalog" + doLast { + def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, "linux", "x86_64") + // Get latest version from catalog (same API call as checkLocalVersionMatchingLatest) + def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${jdkDetails.major}_linux" + def catalogConnection = URI.create(url).toURL().openConnection() + catalogConnection.requestMethod = 'GET' + assert catalogConnection.responseCode == 200 + + def catalogMetadata = new JsonSlurper().parseText(catalogConnection.content.text) + + if (catalogMetadata.version == jdkDetails.revision && catalogMetadata.revision == jdkDetails.build) { + println "JDK version is already up to date: ${jdkDetails.revision} build ${jdkDetails.build}" + return + } + + println "Updating JDK from ${jdkDetails.revision} build ${jdkDetails.build} to ${catalogMetadata.version} build ${catalogMetadata.revision}" + + // Simple line-based replacement preserving format + def versionsFile = file("${projectDir}/versions.yml") + versionsFile.text = versionsFile.text + .replaceAll(/(?m)^(\s*revision:\s*)${Pattern.quote(jdkDetails.revision)}$/, "\$1${catalogMetadata.version}") + .replaceAll(/(?m)^(\s*build:\s*)${jdkDetails.build}$/, "\$1${catalogMetadata.revision}") + } } tasks.register("deleteLocalJdk", Delete) { diff --git a/catalog-info.yaml b/catalog-info.yaml index 143acb634b7..6d7abe61aa3 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -33,7 +33,6 @@ spec: - resource:logstash-windows-jdk-matrix-pipeline - resource:logstash-benchmark-pipeline - resource:logstash-health-report-tests-pipeline - - resource:logstash-jdk-availability-check-pipeline # *********************************** # Declare serverless IT pipeline @@ -744,62 +743,3 @@ spec: branch: main cronline: 30 20 * * * message: Daily trigger of Health Report Tests Pipeline - -# ******************************* -# SECTION END: Health Report Tests pipeline -# ******************************* - -# *********************************** -# Declare JDK check pipeline -# *********************************** ---- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json -apiVersion: backstage.io/v1alpha1 -kind: Resource -metadata: - name: logstash-jdk-availability-check-pipeline - description: ":logstash: check availability of new JDK version" -spec: - type: buildkite-pipeline - owner: group:logstash - system: platform-ingest - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - name: logstash-jdk-availability-check-pipeline - spec: - repository: elastic/logstash - pipeline_file: ".buildkite/jdk_availability_check_pipeline.yml" - maximum_timeout_in_minutes: 10 - provider_settings: - trigger_mode: none # don't trigger jobs from github activity - env: - ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' - SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' - SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' - SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' - teams: - logstash: - access_level: MANAGE_BUILD_AND_READ - ingest-eng-prod: - access_level: MANAGE_BUILD_AND_READ - everyone: - access_level: READ_ONLY - schedules: - Weekly JDK availability check (main): - branch: main - cronline: 0 2 * * 1 # every Monday@2AM UTC - message: Weekly trigger of JDK update availability pipeline per branch - env: - PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline' - Weekly JDK availability check (8.19): - branch: "8.19" - cronline: 0 2 * * 1 # every Monday@2AM UTC - message: Weekly trigger of JDK update availability pipeline per branch - env: - PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline' - -# ******************************* -# SECTION END: JDK check pipeline -# ******************************* diff --git a/ci/check_jdk_version_availability.sh b/ci/check_jdk_version_availability.sh index 2ce40dc7b2f..f854549d022 100755 --- a/ci/check_jdk_version_availability.sh +++ b/ci/check_jdk_version_availability.sh @@ -4,4 +4,4 @@ set -eo pipefail export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" echo "Checking local JDK version against latest remote from JVM catalog" -./gradlew checkNewJdkVersion \ No newline at end of file +./gradlew updateJdkVersion \ No newline at end of file From cf99b52f3abef8354754240942eb2dcf14d9b061 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 24 Sep 2025 11:29:37 -0700 Subject: [PATCH 2/6] Refactor to use updatecli --- .ci/updatecli/bump-java-version.yml | 46 +++++++++++++++++++++++++ .github/workflows/bump-java-version.yml | 30 +++++++++------- 2 files changed, 64 insertions(+), 12 deletions(-) create mode 100644 .ci/updatecli/bump-java-version.yml diff --git a/.ci/updatecli/bump-java-version.yml b/.ci/updatecli/bump-java-version.yml new file mode 100644 index 00000000000..e9955c10887 --- /dev/null +++ b/.ci/updatecli/bump-java-version.yml @@ -0,0 +1,46 @@ +--- +name: Update java version file +pipelineid: "logstash/version-updates-{{ requiredEnv "LOGSTASH_BRANCH" }}" + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + branch: '{{ requiredEnv "LOGSTASH_BRANCH" }}' + commitusingapi: true + force: false + +sources: + latest_jdk_version: + kind: json + spec: + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_21_linux' + key: 'version' + + latest_jdk_build: + kind: json + spec: + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_21_linux' + key: 'revision' + +targets: + update_jdk_revision: + name: "Update JDK revision" + kind: yaml + sourceid: latest_jdk_version + spec: + file: versions.yml + key: $.bundled_jdk.revision + + update_jdk_build: + name: "Update JDK build" + kind: yaml + sourceid: latest_jdk_build + spec: + file: versions.yml + key: $.bundled_jdk.build \ No newline at end of file diff --git a/.github/workflows/bump-java-version.yml b/.github/workflows/bump-java-version.yml index 328870f9eee..818faf236f0 100644 --- a/.github/workflows/bump-java-version.yml +++ b/.github/workflows/bump-java-version.yml @@ -1,19 +1,25 @@ -name: Stub GH action for devoping new workflows [STUB] +name: bump-java-version + on: + schedule: + # Run weekly on Mondays at midnight UTC + - cron: '0 0 * * 1' workflow_dispatch: - pull_request: - types: [opened, synchronize, reopened] -permissions: - pull-requests: write - contents: write jobs: - stub_job_name: - name: Stub Job + bump: + permissions: + contents: write + pull-requests: write runs-on: ubuntu-latest steps: - - name: Stub step - run: | - echo "Stub to iterate via PR" - + - uses: actions/checkout@v5 + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: apply --config .ci/updatecli/bump-java-version.yml --values .ci/updatecli/values.d/scm.yml + version-file: .updatecli-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LOGSTASH_BRANCH: ${{ github.ref_name }} From 58f06b65dcd43ac29213952b1ceca8310b6f652e Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 24 Sep 2025 11:31:46 -0700 Subject: [PATCH 3/6] Remove gradle task in favor of updatecli --- build.gradle | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/build.gradle b/build.gradle index 49e8e4b7803..923bdc49151 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,6 @@ apply from: "rubyUtils.gradle" import de.undercouch.gradle.tasks.download.Download import groovy.json.JsonSlurper -import java.util.regex.Pattern import org.logstash.gradle.tooling.ListProjectDependencies import org.logstash.gradle.tooling.ExtractBundledJdkVersion import org.logstash.gradle.tooling.SignAliasDefinitions @@ -858,33 +857,6 @@ tasks.register("downloadJdk", Download) { } } -tasks.register("updateJdkVersion") { - description = "Update JDK version in versions.yml to match latest from JVM catalog" - doLast { - def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, "linux", "x86_64") - // Get latest version from catalog (same API call as checkLocalVersionMatchingLatest) - def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${jdkDetails.major}_linux" - def catalogConnection = URI.create(url).toURL().openConnection() - catalogConnection.requestMethod = 'GET' - assert catalogConnection.responseCode == 200 - - def catalogMetadata = new JsonSlurper().parseText(catalogConnection.content.text) - - if (catalogMetadata.version == jdkDetails.revision && catalogMetadata.revision == jdkDetails.build) { - println "JDK version is already up to date: ${jdkDetails.revision} build ${jdkDetails.build}" - return - } - - println "Updating JDK from ${jdkDetails.revision} build ${jdkDetails.build} to ${catalogMetadata.version} build ${catalogMetadata.revision}" - - // Simple line-based replacement preserving format - def versionsFile = file("${projectDir}/versions.yml") - versionsFile.text = versionsFile.text - .replaceAll(/(?m)^(\s*revision:\s*)${Pattern.quote(jdkDetails.revision)}$/, "\$1${catalogMetadata.version}") - .replaceAll(/(?m)^(\s*build:\s*)${jdkDetails.build}$/, "\$1${catalogMetadata.revision}") - } -} - tasks.register("deleteLocalJdk", Delete) { // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] String osName = selectOsType() From f2889111b85a24794243d1d1dbf241e5a7c5cda1 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 24 Sep 2025 11:34:04 -0700 Subject: [PATCH 4/6] remove unused file --- ci/check_jdk_version_availability.sh | 7 ------- 1 file changed, 7 deletions(-) delete mode 100755 ci/check_jdk_version_availability.sh diff --git a/ci/check_jdk_version_availability.sh b/ci/check_jdk_version_availability.sh deleted file mode 100755 index f854549d022..00000000000 --- a/ci/check_jdk_version_availability.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" - -echo "Checking local JDK version against latest remote from JVM catalog" -./gradlew updateJdkVersion \ No newline at end of file From b6c6d0c9baa586e7db199b9f92955068f0518158 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 24 Sep 2025 13:09:46 -0700 Subject: [PATCH 5/6] Dynamically generate java version lookup --- .ci/updatecli/bump-java-version.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.ci/updatecli/bump-java-version.yml b/.ci/updatecli/bump-java-version.yml index e9955c10887..9f1bf41feed 100644 --- a/.ci/updatecli/bump-java-version.yml +++ b/.ci/updatecli/bump-java-version.yml @@ -16,16 +16,26 @@ scms: force: false sources: + jdk_major: + kind: yaml + spec: + file: "versions.yml" + key: "$.bundled_jdk.revision" + transformers: + - findsubmatch: + pattern: '^(\d+)\.\d+\.\d+$' + captureindex: 1 + latest_jdk_version: kind: json spec: - file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_21_linux' + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_{{ source "jdk_major" }}_linux' key: 'version' latest_jdk_build: kind: json spec: - file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_21_linux' + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_{{ source "jdk_major" }}_linux' key: 'revision' targets: From 6bc737fac999e3152061b8605402e9905d576774 Mon Sep 17 00:00:00 2001 From: donoghuc Date: Wed, 24 Sep 2025 13:21:03 -0700 Subject: [PATCH 6/6] make pipeline name unique' ' --- .ci/updatecli/bump-java-version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/updatecli/bump-java-version.yml b/.ci/updatecli/bump-java-version.yml index 9f1bf41feed..62f48760d33 100644 --- a/.ci/updatecli/bump-java-version.yml +++ b/.ci/updatecli/bump-java-version.yml @@ -1,6 +1,6 @@ --- name: Update java version file -pipelineid: "logstash/version-updates-{{ requiredEnv "LOGSTASH_BRANCH" }}" +pipelineid: "logstash/jdk-version-updates-{{ requiredEnv "LOGSTASH_BRANCH" }}" scms: default: