Support for Elasticsearch Serverless

.buildkite/create-serverless.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [[ -z $EC_PROJECT_PREFIX ]]; then
+  echo -e "\033[31;1mERROR:\033[0m Required environment variable [EC_PROJECT_PREFIX] not set\033[0m"
+  exit 1
+fi
+
+# Using BUILDKITE_JOB_ID for the name to make it unique:
+export EC_PROJECT_NAME="$EC_PROJECT_PREFIX-$BUILDKITE_JOB_ID"
+echo -e "--- :elasticsearch: Start serverless instance $EC_PROJECT_NAME"
+
+qaf elastic-cloud projects create --project-type elasticsearch
+deployment=$(qaf elastic-cloud projects describe $EC_PROJECT_NAME --as-json --show-credentials)
+
+# Set ELASTICSEARCH_URL and API_KEY variables
+export ES_API_SECRET_KEY=$(echo "$deployment" | jq -r '.credentials.api_key')
+export ELASTICSEARCH_URL=$(echo "$deployment" | jq -r '.elasticsearch.url')
+buildkite-agent meta-data set "ES_API_SECRET_KEY" $ES_API_SECRET_KEY
+buildkite-agent meta-data set "ELASTICSEARCH_URL" $ELASTICSEARCH_URL
+buildkite-agent meta-data set "EC_PROJECT_NAME" $EC_PROJECT_NAME
+
+echo -e "--- :computer: Environment variables"
+echo -e "ELASTICSEARCH_URL $ELASTICSEARCH_URL"

.buildkite/functions/wait-for-container.sh

@@ -14,7 +14,7 @@ function wait_for_container {
     echo ""
     docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
     echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
-    sleep 5;
+    sleep 10;
   done;
 
   # Always show logs if the container is running, this is very useful both on CI as well as while developing

.buildkite/log-results.sh

@@ -14,11 +14,11 @@ for f in $files; do
 
 "
 
-  FAILED_TESTS=`grep -A1 "E,.*" $f`
+  FAILED_TESTS=`grep -A1 "E,.*" $f | sed 's/\#/-/g' | sed 's/^--/\n/g'`
   if [[ -n "$FAILED_TESTS" ]]; then
-    buildkite-agent annotate --append "Failures in $f
-
+    buildkite-agent annotate --append "<details>"
+    buildkite-agent annotate --append "<summary>Failures in $f</summary>
 $FAILED_TESTS
-"
+</details>"
   fi
 done

.buildkite/pipeline.yml

@@ -37,6 +37,52 @@ steps:
       DEBUG: true
     command: ./.buildkite/run-yaml-tests.sh
     artifact_paths: "elasticsearch-api/tmp/*"
+  - label: "Create :elasticsearch: Serverless projects"
+    key: "create-serverless"
+    agents:
+      image: docker.elastic.co/appex-qa/qaf:latest
+    env:
+      EC_PROJECT_PREFIX: ruby
+      EC_REGISTER_BACKEND: buildkite
+      EC_REGION: aws-eu-west-1
+      EC_ENV: qa
+    commands:
+      - mkdir ~/.elastic
+      - touch ~/.elastic/cloud.json
+      - echo "{\"api_key\":{\"qa\":\"$(vault read -field=qa secret/ci/elastic-elasticsearch-ruby/cloud-access)\"}}" > ~/.elastic/cloud.json
+      - ./.buildkite/create-serverless.sh
+  - label: "Run :elasticsearch: Serverless :rspec: Tests :ruby:"
+    key: run-serverless-tests
+    depends_on:
+      - step: create-serverless
+    soft_fail:
+      - exit_status: 1
+    agents:
+      provider: "gcp"
+    env:
+      RUBY_VERSION: "3.4"
+      RUBY_SOURCE: "ruby"
+      TEST_SUITE: serverless
+      ES_YAML_TESTS_BRANCH: main
+      QUIET: false
+    command: ./.buildkite/run-yaml-tests.sh
+    artifact_paths: "elasticsearch-api/tmp/*"
+  - label: "Destroy :elasticsearch: Serverless projects"
+    depends_on:
+      - step: run-serverless-tests
+        allow_failure: true
+    agents:
+      image: docker.elastic.co/appex-qa/qaf:latest
+    env:
+      EC_REGISTER_BACKEND: buildkite
+      EC_ENV: qa
+      EC_REGION: aws-eu-west-1
+    commands:
+      - mkdir ~/.elastic
+      - touch ~/.elastic/cloud.json
+      - export EC_PROJECT_NAME=`buildkite-agent meta-data get "EC_PROJECT_NAME"`
+      - echo "{\"api_key\":{\"qa\":\"$(vault read -field=qa secret/ci/elastic-elasticsearch-ruby/cloud-access)\"}}" > ~/.elastic/cloud.json
+      - qaf elastic-cloud projects delete
   - wait: ~
     continue_on_failure: true
   - label: "Log Results"

.buildkite/run-client.sh → .buildkite/run-serverless-tests.sh

@@ -6,8 +6,11 @@ script_path=$(dirname $(realpath -s $0))
 set -euo pipefail
 repo=`pwd`
 
-export RUBY_VERSION=${RUBY_VERSION:-3.1}
+export RUBY_VERSION=${RUBY_VERSION:-3.4}
+export BUILDKITE=${BUILDKITE:-false}
 export TRANSPORT_VERSION=${TRANSPORT_VERSION:-8}
+ELASTICSEARCH_URL=`buildkite-agent meta-data get "ELASTICSEARCH_URL"`
+ES_API_SECRET_KEY=`buildkite-agent meta-data get "ES_API_SECRET_KEY"`
 
 echo "--- :ruby: Building Docker image"
 docker build \
@@ -23,17 +26,14 @@ mkdir -p elasticsearch-api/tmp
 echo "--- :ruby: Running $TEST_SUITE tests"
 docker run \
        -u "$(id -u)" \
-       --network="${network_name}" \
-       --env "TEST_ES_SERVER=${elasticsearch_url}" \
-       --env "ELASTIC_PASSWORD=${elastic_password}" \
-       --env "TEST_SUITE=${TEST_SUITE}" \
-       --env "ELASTIC_USER=elastic" \
-       --env "BUILDKITE=true" \
-       --env "QUIET=${QUIET}" \
-       --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \
-       --env "STACK_VERSION=${STACK_VERSION}" \
+       -e "ELASTIC_USER=elastic" \
+       -e "QUIET=${QUIET}" \
+       -e "BUILDKITE=${BUILDKITE}" \
+       -e "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \
+       -e "ELASTICSEARCH_URL=${ELASTICSEARCH_URL}" \
+       -e "ES_API_KEY=${ES_API_SECRET_KEY}" \
        --volume $repo:/usr/src/app \
        --name elasticsearch-ruby \
        --rm \
        elastic/elasticsearch-ruby \
-       bundle exec rake es:download_artifacts test:platinum:integration test:rest_api
+       bundle exec bundle exec rake test:yaml

.buildkite/run-yaml-tests.sh

@@ -5,15 +5,27 @@
 # Version 0.1
 #
 script_path=$(dirname $(realpath -s $0))
-source $script_path/functions/imports.sh
+
+if [[ "$TEST_SUITE" == "serverless" ]]; then
+  # Get Elasticsearch Serverless credentials and endpoint
+  export TEST_ES_SERVER=`buildkite-agent meta-data get "ELASTICSEARCH_URL"`
+  export ES_API_SECRET_KEY=`buildkite-agent meta-data get "ES_API_SECRET_KEY"`
+else
+  # Start Elasticsearch Stack on Docker
+  source $script_path/functions/imports.sh
+  echo "--- :elasticsearch: Starting Elasticsearch"
+  DETACH=true bash $script_path/run-elasticsearch.sh
+fi
+
 set -euo pipefail
 repo=`pwd`
 
-echo "--- :elasticsearch: Starting Elasticsearch"
-DETACH=true bash $script_path/run-elasticsearch.sh
-
 export RUBY_VERSION=${RUBY_VERSION:-3.1}
+export BUILDKITE=${BUILDKITE:-false}
 export TRANSPORT_VERSION=${TRANSPORT_VERSION:-8}
+export QUIET=${QUIET:-false}
+export DEBUG=${DEBUG:-false}
+export TEST_SUITE=${TEST_SUITE:-platinum}
 
 echo "--- :ruby: Building Docker image"
 docker build \
@@ -26,20 +38,41 @@ docker build \
 
 mkdir -p elasticsearch-api/tmp
 
-echo "--- :ruby: Running :yaml: tests"
-docker run \
-       -u "$(id -u)" \
-       --network="${network_name}" \
-       --env "TEST_ES_SERVER=${elasticsearch_url}" \
-       --env "ELASTIC_PASSWORD=${elastic_password}" \
-       --env "ELASTIC_USER=elastic" \
-       --env "BUILDKITE=true" \
-       --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \
-       --env "STACK_VERSION=${STACK_VERSION}" \
-       --env "ES_YAML_TESTS_BRANCH=${ES_YAML_TESTS_BRANCH}" \
-       --env "DEBUG=${DEBUG}" \
-       --volume $repo:/usr/src/app \
-       --name elasticsearch-ruby \
-       --rm \
-       elastic/elasticsearch-ruby \
-       bundle exec rake test:yaml
+if [[ "$TEST_SUITE" == "serverless" ]]; then
+  echo "--- :ruby: Running :yaml: tests"
+  docker run \
+         -u "$(id -u)" \
+         --env "TEST_ES_SERVER=${TEST_ES_SERVER}" \
+         --env "ES_API_KEY=${ES_API_SECRET_KEY}" \
+         --env "BUILDKITE=${BUILDKITE}" \
+         --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \
+         --env "ES_YAML_TESTS_BRANCH=${ES_YAML_TESTS_BRANCH}" \
+         --env "TEST_SUITE=${TEST_SUITE}" \
+         --env "DEBUG=${DEBUG}" \
+         --env "QUIET=${QUIET}" \
+         --volume $repo:/usr/src/app \
+         --name elasticsearch-ruby \
+         --rm \
+         elastic/elasticsearch-ruby \
+         bundle exec rake test:yaml
+else
+  echo "--- :ruby: Running stack tests"
+  docker run \
+         -u "$(id -u)" \
+         --network="${network_name}" \
+         --env "TEST_ES_SERVER=${elasticsearch_url}" \
+         --env "ELASTIC_PASSWORD=${elastic_password}" \
+         --env "ELASTIC_USER=elastic" \
+         --env "STACK_VERSION=${STACK_VERSION}" \
+         --env "BUILDKITE=${BUILDKITE}" \
+         --env "TRANSPORT_VERSION=${TRANSPORT_VERSION}" \
+         --env "ES_YAML_TESTS_BRANCH=${ES_YAML_TESTS_BRANCH}" \
+         --env "TEST_SUITE=${TEST_SUITE}" \
+         --env "DEBUG=${DEBUG}" \
+         --env "QUIET=${QUIET}" \
+         --volume $repo:/usr/src/app \
+         --name elasticsearch-ruby \
+         --rm \
+         elastic/elasticsearch-ruby \
+         bundle exec rake test:yaml
+fi

Rakefile

@@ -129,3 +129,12 @@ task :release do
     puts '-' * 80
   end
 end
+
+desc 'Server info'
+task :info do
+  require 'elasticsearch'
+
+  client = Elasticsearch::Client.new(url: ENV['TEST_ES_SERVER'], api_key: ENV['ES_API_KEY'])
+  info = client.info
+  puts "Connected to Elasticsearch cluster #{info['cluster_name']}"
+end

elasticsearch-api/elasticsearch-api.gemspec

@@ -47,7 +47,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'ansi'
   s.add_development_dependency 'bundler'
   s.add_development_dependency 'elasticsearch'
-  s.add_development_dependency 'elasticsearch-test-runner' unless defined?(JRUBY_VERSION) && JRUBY_VERSION <= "9.4"
+  s.add_development_dependency 'elasticsearch-test-runner' unless defined?(JRUBY_VERSION) && JRUBY_VERSION <= '9.4'
   s.add_development_dependency 'minitest'
   s.add_development_dependency 'minitest-reporters', '>= 1.6'
   s.add_development_dependency 'mocha'
@@ -57,6 +57,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'yard'
 
   # Gems for testing integrations
+  s.add_development_dependency 'concurrent-ruby', '1.3.4' if defined? JRUBY_VERSION && JRUBY_VERSION < '9.4'
   s.add_development_dependency 'activesupport'
   s.add_development_dependency 'hashie'
   s.add_development_dependency 'jbuilder'

elasticsearch-api/spec/yaml-test-runner/run.rb

@@ -29,13 +29,17 @@
 password = ENV['ELASTIC_PASSWORD'] || 'changeme'
 uri = URI.parse(host)
 
+def serverless?
+  ENV['TEST_SUITE'] == 'serverless'
+end
+
 if uri.is_a?(URI::HTTPS)
   raw_certificate = File.read("#{CERTS_PATH}/testnode.crt")
   certificate = OpenSSL::X509::Certificate.new(raw_certificate)
   raw_key = File.read("#{CERTS_PATH}/testnode.key")
   key = OpenSSL::PKey::RSA.new(raw_key)
   ca_file = File.expand_path("#{CERTS_PATH}/ca.crt")
-  host = "https://elastic:#{password}@#{uri.host}:#{uri.port}".freeze
+  host = "https://elastic:#{password}@#{uri.host}:#{uri.port}".freeze unless serverless?
   transport_options = {
     ssl: {
       client_cert: certificate,
@@ -49,19 +53,28 @@
   transport_options = {}
 end
 
-CLIENT = if ENV['ES_API_KEY']
-           Elasticsearch::Client.new(host: host, api_key: ENV['ES_API_KEY'], transport_options: transport_options)
-         else
-           Elasticsearch::Client.new(host: host, transport_options: transport_options)
-         end
+options = {
+  host: host,
+  transport_options: transport_options,
+}
+options.merge!({ api_key: ENV['ES_API_KEY'] }) if ENV['ES_API_KEY']
+
+if serverless?
+  options.merge!(
+    {
+      retry_on_status: [409],
+      retry_on_failure: 10,
+      delay_on_retry: 60_000,
+      request_timeout: 120
+    }
+  )
+end
+CLIENT = Elasticsearch::Client.new(options)
 
 tests_path = File.expand_path('./tmp', __dir__)
-ruby_version = if defined? JRUBY_VERSION
-                 "jruby-#{JRUBY_VERSION}"
-               else
-                 "ruby-#{RUBY_VERSION}"
-               end
-log_filename = "es-#{Elasticsearch::VERSION}-transport-#{ENV['TRANSPORT_VERSION']}-#{ruby_version}.log"
+ruby_version = defined?(JRUBY_VERSION) ? "jruby-#{JRUBY_VERSION}" : "ruby-#{RUBY_VERSION}"
+
+log_filename = "es-#{Elasticsearch::VERSION}-#{ENV['TEST_SUITE']}-transport-#{ENV['TRANSPORT_VERSION']}-#{ruby_version}.log"
 logfile = File.expand_path "../../tmp/#{log_filename}", __dir__
 logger = Logger.new(File.open(logfile, 'w'))
 logger.level = ENV['DEBUG'] ? Logger::DEBUG : Logger::WARN
@@ -70,6 +83,7 @@
 current_branch = `git rev-parse --abbrev-ref HEAD`.strip
 branch = current_branch.match(/[0-9]\.[0-9]+/)&.[](0) || ENV['ES_YAML_TESTS_BRANCH'] || nil
 Elasticsearch::Tests::Downloader::run(tests_path, branch)
+
 runner = Elasticsearch::Tests::TestRunner.new(CLIENT, tests_path, logger)
 runner.add_tests_to_skip('knn_search.yml') # TODO: Extract into file
 runner.run(ENV['SINGLE_TEST'] || [])