Merge branch 'main' of https://github.com/elastic/kibana into alertin…

…g/es-recovery-context
elastic · May 27, 2022 · 40149f8 · 40149f8
2 parents 2b03443 + e310b20
commit 40149f8
Show file tree

Hide file tree

Showing 81 changed files with 1,332 additions and 838 deletions.
diff --git a/.buildkite/package-lock.json b/.buildkite/package-lock.json
diff --git a/.buildkite/package.json b/.buildkite/package.json
@@ -3,6 +3,6 @@
   "version": "1.0.0",
   "private": true,
   "dependencies": {
-    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#ae4994aba5f2e72edcc5914e2aa208086e4b7ea3"
+    "kibana-buildkite-library": "git+https://git@github.com/elastic/kibana-buildkite-library#4ecaba35293fb635cf92ca205ee84fca52f19e2e"
   }
 }
diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
@@ -53,6 +53,7 @@ steps:
 
   - command: KIBANA_DOCKER_CONTEXT=cloud .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
+    soft_fail: true
     agents:
       queue: n2-2
     timeout_in_minutes: 30
@@ -74,6 +75,7 @@ steps:
 
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
+    soft_fail: true
     agents:
       queue: n2-2
     timeout_in_minutes: 30

diff --git a/.buildkite/pipelines/es_snapshots/promote.yml b/.buildkite/pipelines/es_snapshots/promote.yml
@@ -12,3 +12,7 @@ steps:
     command: .buildkite/scripts/steps/es_snapshots/promote.sh
     agents:
       queue: kibana-default
+  - wait
+  - trigger: kibana-agent-packer-cache
+    async: true
+    branches: main
diff --git a/.buildkite/scripts/common/env.sh b/.buildkite/scripts/common/env.sh
@@ -7,6 +7,7 @@ export KIBANA_DIR
 export XPACK_DIR="$KIBANA_DIR/x-pack"
 
 export CACHE_DIR="$HOME/.kibana"
+export ES_CACHE_DIR="$HOME/.es-snapshot-cache"
 PARENT_DIR="$(cd "$KIBANA_DIR/.."; pwd)"
 export PARENT_DIR
 export WORKSPACE="${WORKSPACE:-$PARENT_DIR}"

diff --git a/.buildkite/scripts/packer_cache.sh b/.buildkite/scripts/packer_cache.sh
@@ -6,4 +6,10 @@ source .buildkite/scripts/common/util.sh
 source .buildkite/scripts/common/env.sh
 source .buildkite/scripts/common/setup_node.sh
 
+export FORCE_BOOTSTRAP_REMOTE_CACHE=true
+
 yarn kbn bootstrap
+
+for version in $(cat versions.json | jq -r '.versions[].version'); do
+  node scripts/es snapshot --download-only --base-path "$ES_CACHE_DIR" --version "$version"
+done
diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh
@@ -6,12 +6,22 @@ set -euo pipefail
 
 source .buildkite/scripts/steps/artifacts/env.sh
 
-echo "--- Publish Cloud image"
+echo "--- Build and publish Cloud image"
 mkdir -p target
-cd target
 
-buildkite-agent artifact download "kibana-cloud-$FULL_VERSION-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-docker load --input kibana-cloud-$FULL_VERSION-docker-image.tar.gz
+buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+
+node scripts/build \
+  --skip-initialize \
+  --skip-generic-folders \
+  --skip-platform-folders \
+  --skip-archives \
+  --docker-images \
+  --skip-docker-ubi \
+  --skip-docker-ubuntu \
+  --skip-docker-contexts
+
+docker load --input target/kibana-cloud-$FULL_VERSION-docker-image.tar.gz
 
 TAG="$FULL_VERSION-$GIT_COMMIT"
 KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION"
@@ -25,8 +35,6 @@ trap 'docker logout docker.elastic.co' EXIT
 docker push "$KIBANA_TEST_IMAGE"
 docker logout docker.elastic.co
 
-cd -
-
 echo "--- Create deployment"
 CLOUD_DEPLOYMENT_NAME="kibana-artifacts-$TAG"
 

diff --git a/.buildkite/scripts/steps/artifacts/env.sh b/.buildkite/scripts/steps/artifacts/env.sh
@@ -22,7 +22,7 @@ if [[ "$RELEASE_BUILD" == "true" ]]; then
   WORKFLOW="staging"
 else
   FULL_VERSION="$QUALIFIER_VERSION-SNAPSHOT"
-  BUILD_ARGS="--version-qualifier=$VERSION_QUALIFIER"
+  BUILD_ARGS="--skip-docker-cloud --version-qualifier=$VERSION_QUALIFIER"
   WORKFLOW="snapshot"
 fi
 

diff --git a/.buildkite/scripts/steps/checks.sh b/.buildkite/scripts/steps/checks.sh
@@ -6,6 +6,7 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/bootstrap.sh
 
 .buildkite/scripts/steps/checks/precommit_hook.sh
+.buildkite/scripts/steps/checks/ftr_configs.sh
 .buildkite/scripts/steps/checks/bazel_packages.sh
 .buildkite/scripts/steps/checks/telemetry.sh
 .buildkite/scripts/steps/checks/ts_projects.sh

diff --git a/.buildkite/scripts/steps/checks/ftr_configs.sh b/.buildkite/scripts/steps/checks/ftr_configs.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo --- Check FTR Configs
+checks-reporter-with-killswitch "Check FTR Configs" \
+  node scripts/check_ftr_configs
diff --git a/.buildkite/scripts/steps/functional/common.sh b/.buildkite/scripts/steps/functional/common.sh
@@ -9,4 +9,15 @@ source .buildkite/scripts/common/util.sh
 .buildkite/scripts/bootstrap.sh
 .buildkite/scripts/download_build_artifacts.sh
 
+# If cached snapshots are baked into the agent, copy them into our workspace first
+# We are doing this rather than simply changing the ES base path because many workers
+#   run with the workspace mounted in memory or on a local ssd
+cacheDir="$ES_CACHE_DIR/cache"
+if [[ -d "$cacheDir" ]]; then
+  mkdir -p .es/cache
+  echo "--- Copying ES snapshot cache"
+  echo "Copying cached snapshots from $cacheDir to .es/cache"
+  cp -R "$cacheDir"/* .es/cache/
+fi
+
 is_test_execution_step
diff --git a/docs/setup/install/deb.asciidoc b/docs/setup/install/deb.asciidoc
@@ -22,7 +22,7 @@ include::key.asciidoc[]
 
 [source,sh]
 -------------------------
-wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg
 -------------------------
 
 [[deb-repo]]
@@ -49,7 +49,7 @@ ifeval::["{release-state}"=="released"]
 
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------------
-echo "deb https://artifacts.elastic.co/packages/{major-version}/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list
+echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/{major-version}/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-{major-version}.list
 --------------------------------------------------
 
 endif::[]
@@ -58,7 +58,7 @@ ifeval::["{release-state}"=="prerelease"]
 
 ["source","sh",subs="attributes,callouts"]
 --------------------------------------------------
-echo "deb https://artifacts.elastic.co/packages/{major-version}-prerelease/apt stable main" | sudo tee -a /etc/apt/sources.list.d/elastic-{major-version}.list
+echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/{major-version}-prerelease/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-{major-version}.list
 --------------------------------------------------
 
 endif::[]