CI: Artifact based build

.ci/run-elasticsearch.sh

@@ -7,14 +7,16 @@
 # Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
 # Export the NUMBER_OF_NODES variable to start more than 1 node
 
-# Version 1.2.0
+# Version 1.3.0
 # - Initial version of the run-elasticsearch.sh script
 # - Deleting the volume should not dependent on the container still running
 # - Fixed `ES_JAVA_OPTS` config
 # - Moved to STACK_VERSION and TEST_VERSION
 # - Refactored into functions and imports
 # - Support NUMBER_OF_NODES
 # - Added 5 retries on docker pull for fixing transient network errors
+# - Added flags to make local CCR configurations work
+# - Added action.destructive_requires_name=false as the default will be true in v8
 
 script_path=$(dirname $(realpath -s $0))
 source $script_path/functions/imports.sh
@@ -37,6 +39,7 @@ environment=($(cat <<-END
   --env node.attr.testattr=test
   --env path.repo=/tmp
   --env repositories.url.allowed_urls=http://snapshot.test*
+  --env action.destructive_requires_name=false
 END
 ))
 if [[ "$TEST_SUITE" == "platinum" ]]; then
@@ -45,11 +48,11 @@ if [[ "$TEST_SUITE" == "platinum" ]]; then
     --env xpack.license.self_generated.type=trial
     --env xpack.security.enabled=true
     --env xpack.security.http.ssl.enabled=true
-    --env xpack.security.http.ssl.verification_mode=certificate
     --env xpack.security.http.ssl.key=certs/testnode.key
     --env xpack.security.http.ssl.certificate=certs/testnode.crt
     --env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
     --env xpack.security.transport.ssl.enabled=true
+    --env xpack.security.transport.ssl.verification_mode=certificate
     --env xpack.security.transport.ssl.key=certs/testnode.key
     --env xpack.security.transport.ssl.certificate=certs/testnode.crt
     --env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
@@ -104,7 +107,7 @@ END
   docker run \
     --name "$node_name" \
     --network "$network_name" \
-    --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
+    --env "ES_JAVA_OPTS=-Xms1g -Xmx1g -da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions" \
     "${environment[@]}" \
     "${volumes[@]}" \
     --publish "$http_port":9200 \
@@ -124,4 +127,3 @@ END
   fi
 
 done
-

.ci/run-repository.sh

@@ -40,16 +40,10 @@ echo -e "\033[34;1mINFO:\033[0m Retrieving Elasticsearch Version & Hash from con
 ELASTICSEARCH_BUILD_VERSION=$(curl -sSk $external_elasticsearch_url | jq -r '.version.number')
 ELASTICSEARCH_BUILD_HASH=$(curl -sSk $external_elasticsearch_url | jq -r '.version.build_hash')
 
-echo -e "\033[34;1mINFO:\033[0m Downloading Elasticsearch repository @ $ELASTICSEARCH_BUILD_HASH... \033[0m"
-mkdir -p $WORKSPACE/tmp
-curl --retry 10 -sSL "https://github.com/elastic/elasticsearch/archive/$ELASTICSEARCH_BUILD_HASH.zip" > "$WORKSPACE/tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip"
-
-echo -e "\033[34;1mINFO:\033[0m Extracting the Elasticsearch source... \033[0m"
-docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch unzip -q -o "elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip" '*.properties' '*.json' '*.y*ml'
-docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch /bin/sh -c "
-  rm -rf /tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip
-  rm -rf /tmp/elasticsearch/
-  mv /tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH* /tmp/elasticsearch/
+echo -e "\033[34;1mINFO:\033[0m Download Elasticsearch specs... \033[0m"
+docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/go-elasticsearch/internal/build --rm elastic/go-elasticsearch /bin/sh -c "
+  go mod download
+  go run main.go download-spec -o /tmp -d
 "
 
 echo -e "\033[34;1mINFO:\033[0m Execute [$TEST_SUITE] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"

.ci/scripts/tests-free.sh

@@ -11,13 +11,13 @@ rm -rf esapi/test/xpack
 
 echo -e "\033[34;1mINFO:\033[0m Generating the API registry\033[0m"
 
-cd /go-elasticsearch/internal/cmd/generate || exit
+cd /go-elasticsearch/internal/build || exit
 go get -u golang.org/x/tools/cmd/goimports
 PACKAGE_PATH=/go-elasticsearch/esapi go generate ./...
 
 echo -e "\033[34;1mINFO:\033[0m Generating the test files\033[0m"
 
-go run main.go apitests --output '/go-elasticsearch/esapi/test' --input '/tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/test/**/*.y*ml'
+go run main.go apitests --output '/go-elasticsearch/esapi/test' --input "/tmp/rest-api-spec/test/free/**/*.y*ml"
 
 echo -e "\033[34;1mINFO:\033[0m Download tests deps >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
 

.ci/scripts/tests-platinum.sh

@@ -11,14 +11,13 @@ rm -rf rm -rf esapi/test/ml*
 
 echo -e "\033[34;1mINFO:\033[0m Generating the API registry\033[0m"
 
-cd /go-elasticsearch/internal/cmd/generate || exit
+cd /go-elasticsearch/internal/build || exit
 go get -u golang.org/x/tools/cmd/goimports
 PACKAGE_PATH=/go-elasticsearch/esapi go generate ./...
 
 echo -e "\033[34;1mINFO:\033[0m Generating the test files\033[0m"
 
-go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/*.yml'
-go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/**/*.yml'
+go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/rest-api-spec/test/platinum/**/*.yml'
 
 cd /go-elasticsearch || exit
 

Makefile

@@ -1,5 +1,7 @@
 SHELL := /bin/bash
 
+ELASTICSEARCH_DEFAULT_BUILD_VERSION = "8.0.0-SNAPSHOT"
+
 ##@ Test
 test-unit:  ## Run unit tests
 	@printf "\033[2m→ Running unit tests...\033[0m\n"
@@ -149,8 +151,8 @@ lint:  ## Run lint on the package
 	@{ \
 		set -e ; \
 		trap "test -d ../../../.git && git checkout --quiet go.mod" INT TERM EXIT; \
-		echo "cd internal/cmd/generate/ && go vet ./..."; \
-		cd "internal/cmd/generate/" && go mod download && go vet ./...; \
+		echo "cd internal/build/ && go vet ./..."; \
+		cd "internal/build/" && go mod download && go vet ./...; \
 	}
 
 
@@ -266,9 +268,14 @@ godoc: ## Display documentation for the package
 
 cluster: ## Launch an Elasticsearch cluster with Docker
 	$(eval flavor ?= "core")
-	$(eval version ?= "elasticsearch:8.0.0-SNAPSHOT")
 	$(eval elasticsearch_url = "http://es1:9200")
 
+ifdef ELASTICSEARCH_BUILD_VERSION
+	$(eval version ?= "elasticsearch:"${ELASTICSEARCH_BUILD_VERSION})
+else
+	$(eval version ?= "elasticsearch:"${ELASTICSEARCH_DEFAULT_BUILD_VERSION})
+endif
+
 ifeq ($(origin nodes), undefined)
 	$(eval nodes = 1)
 endif
@@ -285,16 +292,14 @@ ifeq ($(flavor), platinum)
 	$(eval xpack_env += --env "xpack.license.self_generated.type=trial")
 	$(eval xpack_env += --env "xpack.security.enabled=true")
 	$(eval xpack_env += --env "xpack.security.http.ssl.enabled=true")
-	$(eval xpack_env += --env "xpack.security.http.ssl.verification_mode=certificate")
 	$(eval xpack_env += --env "xpack.security.http.ssl.key=certs/testnode.key")
 	$(eval xpack_env += --env "xpack.security.http.ssl.certificate=certs/testnode.crt")
 	$(eval xpack_env += --env "xpack.security.http.ssl.certificate_authorities=certs/ca.crt")
-	$(eval xpack_env += --env "xpack.security.http.ssl.verification_mode=none")
 	$(eval xpack_env += --env "xpack.security.transport.ssl.enabled=true")
+	$(eval xpack_env += --env "xpack.security.transport.ssl.verification_mode=certificate")
 	$(eval xpack_env += --env "xpack.security.transport.ssl.key=certs/testnode.key")
 	$(eval xpack_env += --env "xpack.security.transport.ssl.certificate=certs/testnode.crt")
 	$(eval xpack_env += --env "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt")
-	$(eval xpack_env += --env "xpack.security.transport.ssl.verification_mode=none")
 	$(eval xpack_volumes += --volume "$(PWD)/.ci/certs/testnode.crt:/usr/share/elasticsearch/config/certs/testnode.crt")
 	$(eval xpack_volumes += --volume "$(PWD)/.ci/certs/testnode.key:/usr/share/elasticsearch/config/certs/testnode.key")
 	$(eval xpack_volumes += --volume "$(PWD)/.ci/certs/ca.crt:/usr/share/elasticsearch/config/certs/ca.crt")
@@ -345,7 +350,11 @@ ifdef detach
 endif
 
 cluster-update: ## Update the Docker image
-	$(eval version ?= "elasticsearch:8.0.0-SNAPSHOT")
+ifdef ELASTICSEARCH_BUILD_VERSION
+	$(eval version ?= "elasticsearch:"${ELASTICSEARCH_BUILD_VERSION})
+else
+	$(eval version ?= "elasticsearch:"${ELASTICSEARCH_DEFAULT_BUILD_VERSION})
+endif
 	@printf "\033[2m→ Updating the Docker image...\033[0m\n"
 	@docker pull docker.elastic.co/elasticsearch/$(version);
 
@@ -360,48 +369,47 @@ docker: ## Build the Docker image and run it
 
 ##@ Generator
 gen-api:  ## Generate the API package from the JSON specification
-	$(eval input  ?= tmp/elasticsearch)
+	$(eval input  ?= tmp/rest-api-spec)
 	$(eval output ?= esapi)
 ifdef debug
 	$(eval args += --debug)
 endif
 ifdef ELASTICSEARCH_BUILD_VERSION
 	$(eval version = $(ELASTICSEARCH_BUILD_VERSION))
 else
-	$(eval version = $(shell cat "$(input)/buildSrc/version.properties" | grep 'elasticsearch' | cut -d '=' -f 2 | tr -d ' '))
+	$(eval version = $(ELASTICSEARCH_DEFAULT_BUILD_VERSION))
 endif
 ifdef ELASTICSEARCH_BUILD_HASH
 	$(eval build_hash = $(ELASTICSEARCH_BUILD_HASH))
 else
-	$(eval build_hash = $(shell git --git-dir='$(input)/.git' rev-parse --short HEAD))
+	$(eval build_hash = $(shell cat tmp/elasticsearch.json | jq ".projects.elasticsearch.commit_hash"))
 endif
 	@printf "\033[2m→ Generating API package from specification ($(version):$(build_hash))...\033[0m\n"
 	@{ \
 		set -e; \
-		trap "test -d .git && git checkout --quiet $(PWD)/internal/cmd/generate/go.mod" INT TERM EXIT; \
+		trap "test -d .git && git checkout --quiet $(PWD)/internal/build/go.mod" INT TERM EXIT; \
 		export ELASTICSEARCH_BUILD_VERSION=$(version) && \
 		export ELASTICSEARCH_BUILD_HASH=$(build_hash) && \
-		cd internal/cmd/generate && \
-		go run main.go apisource --input '$(PWD)/$(input)/rest-api-spec/src/main/resources/rest-api-spec/api/*.json' --output '$(PWD)/$(output)' $(args) && \
-		go run main.go apisource --input '$(PWD)/$(input)/x-pack/plugin/src/test/resources/rest-api-spec/api/*.json' --output '$(PWD)/$(output)' $(args) && \
+		cd internal/build && \
+		go run main.go apisource --input '$(PWD)/$(input)/api/*.json' --output '$(PWD)/$(output)' $(args) && \
 		go run main.go apistruct --output '$(PWD)/$(output)'; \
 	}
 
 gen-tests:  ## Generate the API tests from the YAML specification
-	$(eval input  ?= tmp/elasticsearch)
+	$(eval input  ?= tmp/rest-api-spec)
 	$(eval output ?= esapi/test)
 ifdef debug
 	$(eval args += --debug)
 endif
 ifdef ELASTICSEARCH_BUILD_VERSION
 	$(eval version = $(ELASTICSEARCH_BUILD_VERSION))
 else
-	$(eval version = $(shell cat "$(input)/buildSrc/version.properties" | grep 'elasticsearch' | cut -d '=' -f 2 | tr -d ' '))
+	$(eval version = $(ELASTICSEARCH_DEFAULT_BUILD_VERSION))
 endif
 ifdef ELASTICSEARCH_BUILD_HASH
 	$(eval build_hash = $(ELASTICSEARCH_BUILD_HASH))
 else
-	$(eval build_hash = $(shell git --git-dir='$(input)/.git' rev-parse --short HEAD))
+	$(eval build_hash = $(shell cat tmp/elasticsearch.json | jq ".projects.elasticsearch.commit_hash"))
 endif
 	@printf "\033[2m→ Generating API tests from specification ($(version):$(build_hash))...\033[0m\n"
 	@{ \
@@ -411,12 +419,11 @@ endif
 		export ELASTICSEARCH_BUILD_HASH=$(build_hash) && \
 		rm -rf $(output)/*_test.go && \
 		rm -rf $(output)/xpack && \
-		cd internal/cmd/generate && \
+		cd internal/build && \
 		go get golang.org/x/tools/cmd/goimports && \
 		go generate ./... && \
-		go run main.go apitests --input '$(PWD)/$(input)/rest-api-spec/src/main/resources/rest-api-spec/test/**/*.y*ml' --output '$(PWD)/$(output)' $(args) && \
-		go run main.go apitests --input '$(PWD)/$(input)/x-pack/plugin/src/test/resources/rest-api-spec/test/**/*.yml' --output '$(PWD)/$(output)/xpack' $(args) && \
-		go run main.go apitests --input '$(PWD)/$(input)/x-pack/plugin/src/test/resources/rest-api-spec/test/**/**/*.yml' --output '$(PWD)/$(output)/xpack' $(args) && \
+		go run main.go apitests --input '$(PWD)/$(input)/test/free/**/*.y*ml' --output '$(PWD)/$(output)' $(args) && \
+		go run main.go apitests --input '$(PWD)/$(input)/test/platinum/**/*.yml' --output '$(PWD)/$(output)/xpack' $(args) && \
 		mkdir -p '$(PWD)/esapi/test/xpack/ml' && \
 		mkdir -p '$(PWD)/esapi/test/xpack/ml-crud' && \
 		mv $(PWD)/esapi/test/xpack/xpack_ml* $(PWD)/esapi/test/xpack/ml/ && \
@@ -445,11 +452,23 @@ gen-docs:  ## Generate the skeleton of documentation examples
 			fi; \
 		) && \
 		printf "\n\033[2m→ Generating ASCIIDoc files from Go source\033[0m\n" && \
-		( cd '$(PWD)/internal/cmd/generate' && \
+		( cd '$(PWD)/internal/build' && \
 			go run main.go examples doc --debug --input='$(PWD)/.doc/examples/src/' --output='$(PWD)/.doc/examples/' \
 		) \
 	}
 
+download-specs: ## Download the latest specs for the specified Elasticsearch version
+	$(eval output ?= tmp)
+	@mkdir -p tmp
+	@{ \
+		set -e; \
+		printf "\n\033[2m→ Downloading latest Elasticsearch specs for version [$(ELASTICSEARCH_DEFAULT_BUILD_VERSION)]\033[0m\n" && \
+		rm -rf $(output)/rest-api-spec && \
+		rm -rf $(output)/elasticsearch.json && \
+		cd internal/build && \
+		go run main.go download-spec --output '$(PWD)/$(output)'; \
+	}
+
 ##@ Other
 #------------------------------------------------------------------------------
 help:  ## Display help

_examples/xkcdsearch/go.sum

@@ -1,3 +1,4 @@
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/rs/zerolog v1.11.0 h1:DRuq/S+4k52uJzBQciUcofXx45GrMC6yrEbb/CoK6+M=
 github.com/rs/zerolog v1.11.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=

internal/cmd/generate/commands/commands.go → internal/build/cmd/commands.go

@@ -2,20 +2,19 @@
 // Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information.
 
-package commands
+package cmd
 
 import (
 	"os"
 
 	"github.com/spf13/cobra"
 
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/utils"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/utils"
 )
 
 var rootCmd = &cobra.Command{
-	Use:   "generate",
-	Short: "generate Go APIs, tests and examples for documentation",
-	// Long:  "TODO",
+	Use:   "build",
+	Short: "Build tools for Elasticsearch client, helpers for CI, etc...",
 }
 
 // Execute launches the CLI application.

internal/cmd/generate/commands/genexamples/command.go → internal/build/cmd/generate/commands/genexamples/command.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/cmd"
 	"io"
 	"io/ioutil"
 	"os"
@@ -20,8 +21,7 @@ import (
 	"github.com/spf13/cobra"
 	"golang.org/x/tools/imports"
 
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/commands"
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/utils"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/utils"
 )
 
 var (
@@ -58,7 +58,7 @@ func init() {
 	genexamplesCmd.AddCommand(genexamplesSrcCmd)
 	genexamplesCmd.AddCommand(genexamplesDocCmd)
 
-	commands.RegisterCmd(genexamplesCmd)
+	cmd.RegisterCmd(genexamplesCmd)
 }
 
 var genexamplesCmd = &cobra.Command{

internal/cmd/generate/commands/genexamples/translator.go → internal/build/cmd/generate/commands/genexamples/translator.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/utils"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/utils"
 )
 
 const testCheck = "\t" + `if err != nil {            // SKIP

internal/cmd/generate/commands/gensource/command.go → internal/build/cmd/generate/commands/gensource/command.go

@@ -7,6 +7,7 @@ package gensource
 import (
 	"bytes"
 	"fmt"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/cmd"
 	"io"
 	"os"
 	"path/filepath"
@@ -15,8 +16,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/commands"
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/utils"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/utils"
 )
 
 var (
@@ -46,7 +46,7 @@ func init() {
 	gensourceCmd.MarkFlagRequired("output")
 	gensourceCmd.Flags().SortFlags = false
 
-	commands.RegisterCmd(gensourceCmd)
+	cmd.RegisterCmd(gensourceCmd)
 }
 
 var gensourceCmd = &cobra.Command{
@@ -142,7 +142,6 @@ func (cmd *Command) Execute() (err error) {
 		if err != nil {
 			return err
 		}
-		defer f.Close()
 
 		if err := cmd.processFile(f); err != nil {
 			return fmt.Errorf("Processing file %q: %s", fname, err)
@@ -153,6 +152,8 @@ func (cmd *Command) Execute() (err error) {
 		if err != nil {
 			return fmt.Errorf("error creating endpoint for %q: %s", fname, err)
 		}
+		f.Close()
+
 		endpoints = append(endpoints, e)
 		stats.n++
 	}

internal/cmd/generate/commands/gensource/debug.go → internal/build/cmd/generate/commands/gensource/debug.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"text/tabwriter"
 
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/utils"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/utils"
 )
 
 // DebugInfo returns information about the endpoint as a string.

internal/cmd/generate/commands/gensource/generator.go → internal/build/cmd/generate/commands/gensource/generator.go

@@ -13,7 +13,7 @@ import (
 
 	"golang.org/x/tools/imports"
 
-	"github.com/elastic/go-elasticsearch/v8/internal/cmd/generate/utils"
+	"github.com/elastic/go-elasticsearch/v8/internal/build/utils"
 )
 
 // Generator represents the "gensource" generator.