Merge branch 'main' into 138015-availability-label

.buildkite/ftr_configs.yml

@@ -117,6 +117,7 @@ enabled:
   - x-pack/test/alerting_api_integration/basic/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group1/config.ts
   - x-pack/test/alerting_api_integration/security_and_spaces/group2/config.ts
+  - x-pack/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
   - x-pack/test/alerting_api_integration/spaces_only/config.ts
   - x-pack/test/api_integration_basic/config.ts
   - x-pack/test/api_integration/config_security_basic.ts

.buildkite/pipeline-utils/test-failures/annotate.ts

@@ -150,7 +150,7 @@ export const annotateTestFailures = async () => {
   }
 
   exec(
-    `buildkite-agent artifact download --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
+    `.buildkite/scripts/common/download_artifact.sh --include-retried-jobs "target/test_failures/*.json" "${failureDir}"`
   );
 
   const failures: TestFailure[] = recursiveReadDir(failureDir)

.buildkite/scripts/common/download_artifact.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(dirname "$0")/util.sh"
+download_artifact "$@"

.buildkite/scripts/common/util.sh

@@ -168,3 +168,9 @@ npm_install_global() {
     npm install -g "$toInstall" && touch "$successFlag"
   fi
 }
+
+# Download an artifact using the buildkite-agent, takes the same arguments as https://buildkite.com/docs/agent/v3/cli-artifact#downloading-artifacts-usage
+# times-out after 60 seconds and retries up to 3 times
+download_artifact() {
+  retry 3 1 timeout 3m buildkite-agent artifact download "$@"
+}

.buildkite/scripts/download_build_artifacts.sh

@@ -2,13 +2,15 @@
 
 set -euo pipefail
 
+source "$(dirname "$0")/common/util.sh"
+
 if [[ ! -d "$KIBANA_BUILD_LOCATION/bin" ]]; then
   echo '--- Downloading Distribution and Plugin artifacts'
 
   cd "$WORKSPACE"
 
-  buildkite-agent artifact download kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-  buildkite-agent artifact download kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact kibana-default.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact kibana-default-plugins.tar.gz . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
   mkdir -p "$KIBANA_BUILD_LOCATION"
   tar -xzf kibana-default.tar.gz -C "$KIBANA_BUILD_LOCATION" --strip=1

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -4,12 +4,13 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
+source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/artifacts/env.sh
 
 echo "--- Build and publish Cloud image"
 mkdir -p target
 
-buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 node scripts/build \
   --skip-initialize \

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -4,6 +4,7 @@ set -euo pipefail
 
 .buildkite/scripts/bootstrap.sh
 
+source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/artifacts/env.sh
 
 KIBANA_DOCKER_CONTEXT="${KIBANA_DOCKER_CONTEXT:="default"}"
@@ -28,7 +29,7 @@ fi
 tar -xf "target/$DOCKER_CONTEXT_FILE" -C "$DOCKER_BUILD_FOLDER"
 cd $DOCKER_BUILD_FOLDER
 
-buildkite-agent artifact download "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build context"
 docker build .

.buildkite/scripts/steps/artifacts/publish.sh

@@ -7,8 +7,8 @@ source .buildkite/scripts/steps/artifacts/env.sh
 
 echo "--- Download and verify artifacts"
 function download {
-  buildkite-agent artifact download "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
-  buildkite-agent artifact download "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact "$1.sha512.txt" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   sha512sum -c "$1.sha512.txt"
   rm "$1.sha512.txt"
 }
@@ -40,7 +40,7 @@ download "kibana-$FULL_VERSION-windows-x86_64.zip"
 
 download "dependencies-$FULL_VERSION.csv"
 
-cd - 
+cd -
 
 echo "--- Set artifact permissions"
 chmod -R a+r target/*

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -14,7 +14,7 @@ ECCTL_LOGS=$(mktemp --suffix ".json")
 echo "--- Download Kibana Distribution"
 
 mkdir -p ./target
-buildkite-agent artifact download "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+download_artifact "kibana-$VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
 echo "--- Build Cloud Distribution"
 ELASTICSEARCH_MANIFEST_URL="https://storage.googleapis.com/kibana-ci-es-snapshots-daily/$(jq -r '.version' package.json)/manifest-latest-verified.json"

.buildkite/scripts/steps/code_coverage/ftr_configs.sh

@@ -28,7 +28,7 @@ configs="${FTR_CONFIG:-}"
 
 if [[ "$configs" == "" ]]; then
   echo "--- Downloading ftr test run order"
-  buildkite-agent artifact download ftr_run_order.json .
+  download_artifact ftr_run_order.json .
   configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
 fi
 

.buildkite/scripts/steps/code_coverage/ingest.sh

@@ -29,7 +29,7 @@ revolveBuildHashes() {
 }
 
 collectRan() {
-  buildkite-agent artifact download target/ran_files/* .
+  download_artifact target/ran_files/* .
 
   while read -r x; do
     ran=("${ran[@]}" "$(cat "$x")")
@@ -51,7 +51,7 @@ fetchArtifacts() {
 
   local xs=("$@")
   for x in "${xs[@]}"; do
-    buildkite-agent artifact download "target/kibana-coverage/${x}/*" .
+    download_artifact "target/kibana-coverage/${x}/*" .
   done
 }
 

.buildkite/scripts/steps/code_coverage/jest_parallel.sh

@@ -2,6 +2,7 @@
 
 set -euo pipefail
 
+source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/code_coverage/util.sh
 
 export JOB=$BUILDKITE_PARALLEL_JOB
@@ -18,7 +19,7 @@ fi
 
 export TEST_TYPE
 echo "--- downloading jest test run order"
-buildkite-agent artifact download jest_run_order.json .
+download_artifact jest_run_order.json .
 configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
 
 echo "--- KIBANA_DIR: $KIBANA_DIR"

.buildkite/scripts/steps/functional/scalability_dataset_extraction.sh

@@ -34,8 +34,8 @@ mkdir "${BUILD_ID}"
 tar -czf "${BUILD_ID}/scalability_traces.tar.gz" -C target scalability_traces
 buildkite-agent artifact upload "${BUILD_ID}/scalability_traces.tar.gz"
 # Upload Kibana build, plugins, commit sha and traces to the bucket
-buildkite-agent artifact download kibana-default.tar.gz ./"${BUILD_ID}"
-buildkite-agent artifact download kibana-default-plugins.tar.gz ./"${BUILD_ID}"
+download_artifact kibana-default.tar.gz ./"${BUILD_ID}"
+download_artifact kibana-default-plugins.tar.gz ./"${BUILD_ID}"
 echo "${BUILDKITE_COMMIT}" > "${BUILD_ID}/KIBANA_COMMIT_HASH"
 gsutil -m cp -r "${BUILD_ID}" "${GCS_BUCKET}"
 echo "--- Update reference to the latest CI build"

.buildkite/scripts/steps/package_testing/test.sh

@@ -2,20 +2,21 @@
 
 set -euo pipefail
 
+source "$(dirname "$0")/../../common/util.sh"
 .buildkite/scripts/bootstrap.sh
 
 echo "--- Package Testing for $TEST_PACKAGE"
 
 mkdir -p target
 cd target
 if [[ "$TEST_PACKAGE" == "deb" ]]; then
-  buildkite-agent artifact download 'kibana-*-amd64.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact 'kibana-*-amd64.deb' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.5"
 elif [[ "$TEST_PACKAGE" == "rpm" ]]; then
-  buildkite-agent artifact download 'kibana-*-x86_64.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact 'kibana-*-x86_64.rpm' . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.6"
 elif [[ "$TEST_PACKAGE" == "docker" ]]; then
-  buildkite-agent artifact download "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
+  download_artifact "kibana-$KIBANA_PKG_VERSION*-docker-image.tar.gz" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
   KIBANA_IP_ADDRESS="192.168.56.7"
 fi
 cd ..

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -32,7 +32,7 @@ fi
 
 if [ "$configs" == "" ] && [ "$FTR_CONFIG_GROUP_KEY" != "" ]; then
   echo "--- downloading ftr test run order"
-  buildkite-agent artifact download ftr_run_order.json .
+  download_artifact ftr_run_order.json .
   configs=$(jq -r '.[env.FTR_CONFIG_GROUP_KEY].names[]' ftr_run_order.json)
 fi
 

.buildkite/scripts/steps/test/jest_parallel.sh

@@ -2,6 +2,7 @@
 
 set -euo pipefail
 
+source "$(dirname "$0")/../../common/util.sh"
 export JOB=${BUILDKITE_PARALLEL_JOB:-0}
 
 # a jest failure will result in the script returning an exit code of 10
@@ -20,7 +21,7 @@ fi
 
 export TEST_TYPE
 echo "--- downloading jest test run order"
-buildkite-agent artifact download jest_run_order.json .
+download_artifact jest_run_order.json .
 configs=$(jq -r 'getpath([env.TEST_TYPE]) | .groups[env.JOB | tonumber].names | .[]' jest_run_order.json)
 
 while read -r config; do