From 3a8f98d3228e8304428e8b0f51f2f312dd75ac02 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Wed, 26 Nov 2025 17:14:22 +0000 Subject: [PATCH 1/3] [CI] Replaces deprecated artifacts API with new one --- rake_tasks/elasticsearch_tasks.rake | 32 +++++++++++++++-------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/rake_tasks/elasticsearch_tasks.rake b/rake_tasks/elasticsearch_tasks.rake index 316cbb71ae..f91a23a30f 100644 --- a/rake_tasks/elasticsearch_tasks.rake +++ b/rake_tasks/elasticsearch_tasks.rake @@ -74,36 +74,38 @@ namespace :es do end # Deprecated - desc 'Download Elasticsearch artifacts (tests and REST spec) for currently running cluster' + desc 'Download Elasticsearch artifacts (tests and REST spec) for a given version' task :download_artifacts, :version do |_, args| - json_filename = CURRENT_PATH.join('tmp/artifacts.json') + require 'net/http' + json_filename = CURRENT_PATH.join('tmp/artifacts.json') version_number = args[:version] || ENV['STACK_VERSION'] || version_from_buildkite || version_from_running_cluster # Create ./tmp if it doesn't exist Dir.mkdir(CURRENT_PATH.join('tmp'), 0700) unless File.directory?(CURRENT_PATH.join('tmp')) - # Download json file with package information for version: - json_url = "https://artifacts-api.elastic.co/v1/versions/#{version_number}" - download_file!(json_url, json_filename) + # TODO: If required, bring back the functionality to download a specific build. + # This was implemented in the previous version of the API, but research is needed to see how it + # works in new API. + # + # Get the latest minor from version number, and get latest snapshot + major_minor = version_number.split('.')[0..1].join('.') + url = URI("https://artifacts-snapshot.elastic.co/elasticsearch/latest/#{major_minor}.json") + manifest_url = JSON.parse(Net::HTTP.get(url))['manifest_url'] + download_file!(manifest_url, json_filename) # Parse the downloaded JSON begin artifacts = JSON.parse(File.read(json_filename)) rescue StandardError => e - STDERR.puts "[!] Couldn't read JSON file #{json_filename}" + warn "[!] Couldn't read JSON file #{json_filename}\n#{e.message}" exit 1 end - # Either find the artifacts for the exact same build hash from the current running cluster or - # use the first one from the list of builds: - build_hash_artifact = artifacts['version']['builds'].find do |build| - build.dig('projects', 'elasticsearch', 'commit_hash') == @build_hash - end || artifacts['version']['builds'].first - zip_url = build_hash_artifact.dig('projects', 'elasticsearch', 'packages').select { |k, _| k =~ /rest-resources-zip/ }.map { |_, v| v['url'] }.first - - # Dig into the elasticsearch packages, search for the rest-resources-zip package and return the URL: - build_hash_artifact.dig('projects', 'elasticsearch', 'packages').select { |k, _| k =~ /rest-resources-zip/ }.map { |_, v| v['url'] }.first + # Search the JSON for the rest-resources-zip file and get the URL + packages = artifacts.dig('projects', 'elasticsearch', 'packages') + rest_resources = packages.select { |k, v| k =~ /rest-resources/ } + zip_url = rest_resources.map { |_, v| v['url'] }.first # Download the zip file filename = CURRENT_PATH.join("tmp/#{zip_url.split('/').last}") From 8d99f0d485a29448570c047a08196ce36f7276f9 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Wed, 26 Nov 2025 17:23:23 +0000 Subject: [PATCH 2/3] [CI] Bumps webmock development dependency --- elasticsearch/elasticsearch.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticsearch/elasticsearch.gemspec b/elasticsearch/elasticsearch.gemspec index 29bce4461e..a8183d407e 100644 --- a/elasticsearch/elasticsearch.gemspec +++ b/elasticsearch/elasticsearch.gemspec @@ -59,7 +59,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'rspec' s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius) s.add_development_dependency 'simplecov' - s.add_development_dependency 'webmock' + s.add_development_dependency 'webmock', '> 3.23' s.add_development_dependency 'yard' s.description = <<-DESC.gsub(/^ /, '') From f4ce96c9a2e2e43a0bca20fe5460129c141a2d69 Mon Sep 17 00:00:00 2001 From: Fernando Briano Date: Wed, 26 Nov 2025 17:28:51 +0000 Subject: [PATCH 3/3] [CI] Adjusts downloading artifacts for main --- .github/workflows/main.yml | 2 +- .github/workflows/otel.yml | 2 +- rake_tasks/elasticsearch_tasks.rake | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d7d027fe8c..668ed52e37 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,4 +41,4 @@ jobs: - name: elasticsearch run: cd elasticsearch && bundle exec rake test:all - name: elasticsearch-api - run: rake es:download_artifacts[9.3.0-SNAPSHOT] && cd elasticsearch-api && bundle exec rake test:all + run: rake es:download_artifacts[main] && cd elasticsearch-api && bundle exec rake test:all diff --git a/.github/workflows/otel.yml b/.github/workflows/otel.yml index fc9d926b0b..5a328d9cfb 100644 --- a/.github/workflows/otel.yml +++ b/.github/workflows/otel.yml @@ -43,4 +43,4 @@ jobs: - name: elasticsearch run: cd elasticsearch && bundle exec rake test:all - name: elasticsearch-api - run: rake es:download_artifacts[9.3.0-SNAPSHOT] && cd elasticsearch-api && bundle exec rake test:all + run: rake es:download_artifacts[main] && cd elasticsearch-api && bundle exec rake test:all diff --git a/rake_tasks/elasticsearch_tasks.rake b/rake_tasks/elasticsearch_tasks.rake index f91a23a30f..91981f085c 100644 --- a/rake_tasks/elasticsearch_tasks.rake +++ b/rake_tasks/elasticsearch_tasks.rake @@ -89,8 +89,13 @@ namespace :es do # works in new API. # # Get the latest minor from version number, and get latest snapshot - major_minor = version_number.split('.')[0..1].join('.') + major_minor = if version_number == 'main' + 'master' + else + version_number.split('.')[0..1].join('.') + end url = URI("https://artifacts-snapshot.elastic.co/elasticsearch/latest/#{major_minor}.json") + manifest_url = JSON.parse(Net::HTTP.get(url))['manifest_url'] download_file!(manifest_url, json_filename)