diff --git a/.buildkite/create-serverless.sh b/.buildkite/create-serverless.sh new file mode 100755 index 0000000000..7b7d2c1dcf --- /dev/null +++ b/.buildkite/create-serverless.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ -z $EC_PROJECT_PREFIX ]]; then + echo -e "\033[31;1mERROR:\033[0m Required environment variable [EC_PROJECT_PREFIX] not set\033[0m" + exit 1 +fi + +# Using BUILDKITE_JOB_ID for the name to make it unique: +export EC_PROJECT_NAME="$EC_PROJECT_PREFIX-$BUILDKITE_JOB_ID" +echo -e "--- :elasticsearch: Start serverless instance $EC_PROJECT_NAME" + +qaf elastic-cloud projects create --project-type elasticsearch +deployment=$(qaf elastic-cloud projects describe $EC_PROJECT_NAME --as-json --show-credentials) + +# Set ELASTICSEARCH_URL and API_KEY variables +export ES_API_SECRET_KEY=$(echo "$deployment" | jq -r '.credentials.api_key') +export ELASTICSEARCH_URL=$(echo "$deployment" | jq -r '.elasticsearch.url') +buildkite-agent meta-data set "ES_API_SECRET_KEY" $ES_API_SECRET_KEY +buildkite-agent meta-data set "ELASTICSEARCH_URL" $ELASTICSEARCH_URL +buildkite-agent meta-data set "EC_PROJECT_NAME" $EC_PROJECT_NAME + +echo -e "--- :computer: Environment variables" +echo -e "ELASTICSEARCH_URL $ELASTICSEARCH_URL" diff --git a/.buildkite/functions/wait-for-container.sh b/.buildkite/functions/wait-for-container.sh index 32a4498f46..89d65f4e7d 100755 --- a/.buildkite/functions/wait-for-container.sh +++ b/.buildkite/functions/wait-for-container.sh @@ -14,7 +14,7 @@ function wait_for_container { echo "" docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1} echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m" - sleep 5; + sleep 10; done; # Always show logs if the container is running, this is very useful both on CI as well as while developing diff --git a/.buildkite/log-results.sh b/.buildkite/log-results.sh index 9481b6b5ce..2ee3d01b3a 100755 --- a/.buildkite/log-results.sh +++ b/.buildkite/log-results.sh @@ -14,11 +14,11 @@ for f in $files; do " - FAILED_TESTS=`grep -A1 "E,.*" $f` + FAILED_TESTS=`grep -A1 "E,.*" $f | sed 's/\#/-/g' | sed 's/^--/\n/g'` if [[ -n "$FAILED_TESTS" ]]; then - buildkite-agent annotate --append "Failures in $f - + buildkite-agent annotate --append "
" + buildkite-agent annotate --append "Failures in $f $FAILED_TESTS -" +
" fi done diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index ba30184799..5696edca12 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -37,6 +37,52 @@ steps: DEBUG: true command: ./.buildkite/run-yaml-tests.sh artifact_paths: "elasticsearch-api/tmp/*" + - label: "Create :elasticsearch: Serverless projects" + key: "create-serverless" + agents: + image: docker.elastic.co/appex-qa/qaf:latest + env: + EC_PROJECT_PREFIX: ruby + EC_REGISTER_BACKEND: buildkite + EC_REGION: aws-eu-west-1 + EC_ENV: qa + commands: + - mkdir ~/.elastic + - touch ~/.elastic/cloud.json + - echo "{\"api_key\":{\"qa\":\"$(vault read -field=qa secret/ci/elastic-elasticsearch-ruby/cloud-access)\"}}" > ~/.elastic/cloud.json + - ./.buildkite/create-serverless.sh + - label: "Run :elasticsearch: Serverless :rspec: Tests :ruby:" + key: run-serverless-tests + depends_on: + - step: create-serverless + soft_fail: + - exit_status: 1 + agents: + provider: "gcp" + env: + RUBY_VERSION: "3.4" + RUBY_SOURCE: "ruby" + TEST_SUITE: serverless + ES_YAML_TESTS_BRANCH: main + QUIET: false + command: ./.buildkite/run-yaml-tests.sh + artifact_paths: "elasticsearch-api/tmp/*" + - label: "Destroy :elasticsearch: Serverless projects" + depends_on: + - step: run-serverless-tests + allow_failure: true + agents: + image: docker.elastic.co/appex-qa/qaf:latest + env: + EC_REGISTER_BACKEND: buildkite + EC_ENV: qa + EC_REGION: aws-eu-west-1 + commands: + - mkdir ~/.elastic + - touch ~/.elastic/cloud.json + - export EC_PROJECT_NAME=`buildkite-agent meta-data get "EC_PROJECT_NAME"` + - echo "{\"api_key\":{\"qa\":\"$(vault read -field=qa secret/ci/elastic-elasticsearch-ruby/cloud-access)\"}}" > ~/.elastic/cloud.json + - qaf elastic-cloud projects delete - wait: ~ continue_on_failure: true - label: "Log Results" diff --git a/.buildkite/run-client.sh b/.buildkite/run-serverless-tests.sh similarity index 58% rename from .buildkite/run-client.sh rename to .buildkite/run-serverless-tests.sh index 01287fe78b..d145d6e6d3 100755 --- a/.buildkite/run-client.sh +++ b/.buildkite/run-serverless-tests.sh @@ -6,8 +6,11 @@ script_path=$(dirname $(realpath -s $0)) set -euo pipefail repo=`pwd` -export RUBY_VERSION=${RUBY_VERSION:-3.1} +export RUBY_VERSION=${RUBY_VERSION:-3.4} +export BUILDKITE=${BUILDKITE:-false} export TRANSPORT_VERSION=${TRANSPORT_VERSION:-8} +ELASTICSEARCH_URL=`buildkite-agent meta-data get "ELASTICSEARCH_URL"` +ES_API_SECRET_KEY=`buildkite-agent meta-data get "ES_API_SECRET_KEY"` echo "--- :ruby: Building Docker image" docker build \ @@ -23,17 +26,14 @@ mkdir -p elasticsearch-api/tmp echo "--- :ruby: Running $TEST_SUITE tests" docker run \ -u "$(id -u)" \ - --network="${network_name}" \ - --env "TEST_ES_SERVER=${elasticsearch_url}" \ - --env "ELASTIC_PASSWORD=${elastic_password}" \ - --env "TEST_SUITE=${TEST_SUITE}" \ - --env "ELASTIC_USER=elastic" \ - --env "BUILDKITE=true" \ - --env "QUIET=${QUIET}" \ - --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \ - --env "STACK_VERSION=${STACK_VERSION}" \ + -e "ELASTIC_USER=elastic" \ + -e "QUIET=${QUIET}" \ + -e "BUILDKITE=${BUILDKITE}" \ + -e "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \ + -e "ELASTICSEARCH_URL=${ELASTICSEARCH_URL}" \ + -e "ES_API_KEY=${ES_API_SECRET_KEY}" \ --volume $repo:/usr/src/app \ --name elasticsearch-ruby \ --rm \ elastic/elasticsearch-ruby \ - bundle exec rake es:download_artifacts test:platinum:integration test:rest_api + bundle exec bundle exec rake test:yaml diff --git a/.buildkite/run-yaml-tests.sh b/.buildkite/run-yaml-tests.sh old mode 100644 new mode 100755 index a463972037..59faabb19c --- a/.buildkite/run-yaml-tests.sh +++ b/.buildkite/run-yaml-tests.sh @@ -5,15 +5,27 @@ # Version 0.1 # script_path=$(dirname $(realpath -s $0)) -source $script_path/functions/imports.sh + +if [[ "$TEST_SUITE" == "serverless" ]]; then + # Get Elasticsearch Serverless credentials and endpoint + export TEST_ES_SERVER=`buildkite-agent meta-data get "ELASTICSEARCH_URL"` + export ES_API_SECRET_KEY=`buildkite-agent meta-data get "ES_API_SECRET_KEY"` +else + # Start Elasticsearch Stack on Docker + source $script_path/functions/imports.sh + echo "--- :elasticsearch: Starting Elasticsearch" + DETACH=true bash $script_path/run-elasticsearch.sh +fi + set -euo pipefail repo=`pwd` -echo "--- :elasticsearch: Starting Elasticsearch" -DETACH=true bash $script_path/run-elasticsearch.sh - export RUBY_VERSION=${RUBY_VERSION:-3.1} +export BUILDKITE=${BUILDKITE:-false} export TRANSPORT_VERSION=${TRANSPORT_VERSION:-8} +export QUIET=${QUIET:-false} +export DEBUG=${DEBUG:-false} +export TEST_SUITE=${TEST_SUITE:-platinum} echo "--- :ruby: Building Docker image" docker build \ @@ -26,20 +38,41 @@ docker build \ mkdir -p elasticsearch-api/tmp -echo "--- :ruby: Running :yaml: tests" -docker run \ - -u "$(id -u)" \ - --network="${network_name}" \ - --env "TEST_ES_SERVER=${elasticsearch_url}" \ - --env "ELASTIC_PASSWORD=${elastic_password}" \ - --env "ELASTIC_USER=elastic" \ - --env "BUILDKITE=true" \ - --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \ - --env "STACK_VERSION=${STACK_VERSION}" \ - --env "ES_YAML_TESTS_BRANCH=${ES_YAML_TESTS_BRANCH}" \ - --env "DEBUG=${DEBUG}" \ - --volume $repo:/usr/src/app \ - --name elasticsearch-ruby \ - --rm \ - elastic/elasticsearch-ruby \ - bundle exec rake test:yaml +if [[ "$TEST_SUITE" == "serverless" ]]; then + echo "--- :ruby: Running :yaml: tests" + docker run \ + -u "$(id -u)" \ + --env "TEST_ES_SERVER=${TEST_ES_SERVER}" \ + --env "ES_API_KEY=${ES_API_SECRET_KEY}" \ + --env "BUILDKITE=${BUILDKITE}" \ + --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \ + --env "ES_YAML_TESTS_BRANCH=${ES_YAML_TESTS_BRANCH}" \ + --env "TEST_SUITE=${TEST_SUITE}" \ + --env "DEBUG=${DEBUG}" \ + --env "QUIET=${QUIET}" \ + --volume $repo:/usr/src/app \ + --name elasticsearch-ruby \ + --rm \ + elastic/elasticsearch-ruby \ + bundle exec rake test:yaml +else + echo "--- :ruby: Running stack tests" + docker run \ + -u "$(id -u)" \ + --network="${network_name}" \ + --env "TEST_ES_SERVER=${elasticsearch_url}" \ + --env "ELASTIC_PASSWORD=${elastic_password}" \ + --env "ELASTIC_USER=elastic" \ + --env "STACK_VERSION=${STACK_VERSION}" \ + --env "BUILDKITE=${BUILDKITE}" \ + --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \ + --env "ES_YAML_TESTS_BRANCH=${ES_YAML_TESTS_BRANCH}" \ + --env "TEST_SUITE=${TEST_SUITE}" \ + --env "DEBUG=${DEBUG}" \ + --env "QUIET=${QUIET}" \ + --volume $repo:/usr/src/app \ + --name elasticsearch-ruby \ + --rm \ + elastic/elasticsearch-ruby \ + bundle exec rake test:yaml +fi diff --git a/Rakefile b/Rakefile index 3d1cbacdea..62a6205a2b 100644 --- a/Rakefile +++ b/Rakefile @@ -129,3 +129,12 @@ task :release do puts '-' * 80 end end + +desc 'Server info' +task :info do + require 'elasticsearch' + + client = Elasticsearch::Client.new(url: ENV['TEST_ES_SERVER'], api_key: ENV['ES_API_KEY']) + info = client.info + puts "Connected to Elasticsearch cluster #{info['cluster_name']}" +end diff --git a/elasticsearch-api/elasticsearch-api.gemspec b/elasticsearch-api/elasticsearch-api.gemspec index 73f8917072..f576910773 100644 --- a/elasticsearch-api/elasticsearch-api.gemspec +++ b/elasticsearch-api/elasticsearch-api.gemspec @@ -47,7 +47,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'ansi' s.add_development_dependency 'bundler' s.add_development_dependency 'elasticsearch' - s.add_development_dependency 'elasticsearch-test-runner' unless defined?(JRUBY_VERSION) && JRUBY_VERSION <= "9.4" + s.add_development_dependency 'elasticsearch-test-runner' unless defined?(JRUBY_VERSION) && JRUBY_VERSION <= '9.4' s.add_development_dependency 'minitest' s.add_development_dependency 'minitest-reporters', '>= 1.6' s.add_development_dependency 'mocha' @@ -57,6 +57,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'yard' # Gems for testing integrations + s.add_development_dependency 'concurrent-ruby', '1.3.4' if defined? JRUBY_VERSION && JRUBY_VERSION < '9.4' s.add_development_dependency 'activesupport' s.add_development_dependency 'hashie' s.add_development_dependency 'jbuilder' diff --git a/elasticsearch-api/spec/yaml-test-runner/run.rb b/elasticsearch-api/spec/yaml-test-runner/run.rb index d1896627f6..701d26b801 100644 --- a/elasticsearch-api/spec/yaml-test-runner/run.rb +++ b/elasticsearch-api/spec/yaml-test-runner/run.rb @@ -29,13 +29,17 @@ password = ENV['ELASTIC_PASSWORD'] || 'changeme' uri = URI.parse(host) +def serverless? + ENV['TEST_SUITE'] == 'serverless' +end + if uri.is_a?(URI::HTTPS) raw_certificate = File.read("#{CERTS_PATH}/testnode.crt") certificate = OpenSSL::X509::Certificate.new(raw_certificate) raw_key = File.read("#{CERTS_PATH}/testnode.key") key = OpenSSL::PKey::RSA.new(raw_key) ca_file = File.expand_path("#{CERTS_PATH}/ca.crt") - host = "https://elastic:#{password}@#{uri.host}:#{uri.port}".freeze + host = "https://elastic:#{password}@#{uri.host}:#{uri.port}".freeze unless serverless? transport_options = { ssl: { client_cert: certificate, @@ -49,19 +53,28 @@ transport_options = {} end -CLIENT = if ENV['ES_API_KEY'] - Elasticsearch::Client.new(host: host, api_key: ENV['ES_API_KEY'], transport_options: transport_options) - else - Elasticsearch::Client.new(host: host, transport_options: transport_options) - end +options = { + host: host, + transport_options: transport_options, +} +options.merge!({ api_key: ENV['ES_API_KEY'] }) if ENV['ES_API_KEY'] + +if serverless? + options.merge!( + { + retry_on_status: [409], + retry_on_failure: 10, + delay_on_retry: 60_000, + request_timeout: 120 + } + ) +end +CLIENT = Elasticsearch::Client.new(options) tests_path = File.expand_path('./tmp', __dir__) -ruby_version = if defined? JRUBY_VERSION - "jruby-#{JRUBY_VERSION}" - else - "ruby-#{RUBY_VERSION}" - end -log_filename = "es-#{Elasticsearch::VERSION}-transport-#{ENV['TRANSPORT_VERSION']}-#{ruby_version}.log" +ruby_version = defined?(JRUBY_VERSION) ? "jruby-#{JRUBY_VERSION}" : "ruby-#{RUBY_VERSION}" + +log_filename = "es-#{Elasticsearch::VERSION}-#{ENV['TEST_SUITE']}-transport-#{ENV['TRANSPORT_VERSION']}-#{ruby_version}.log" logfile = File.expand_path "../../tmp/#{log_filename}", __dir__ logger = Logger.new(File.open(logfile, 'w')) logger.level = ENV['DEBUG'] ? Logger::DEBUG : Logger::WARN @@ -70,6 +83,7 @@ current_branch = `git rev-parse --abbrev-ref HEAD`.strip branch = current_branch.match(/[0-9]\.[0-9]+/)&.[](0) || ENV['ES_YAML_TESTS_BRANCH'] || nil Elasticsearch::Tests::Downloader::run(tests_path, branch) + runner = Elasticsearch::Tests::TestRunner.new(CLIENT, tests_path, logger) runner.add_tests_to_skip('knn_search.yml') # TODO: Extract into file runner.run(ENV['SINGLE_TEST'] || [])