Merge branch 'main' into task/endpoint-exceptions-is-wildcard

.buildkite/pipelines/artifacts.yml

@@ -6,7 +6,7 @@ steps:
       imageProject: elastic-images-qa
       provider: gcp
       machineType: c2-standard-16
-    timeout_in_minutes: 60
+    timeout_in_minutes: 75
     retry:
       automatic:
         - exit_status: '*'
@@ -92,6 +92,19 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: KIBANA_DOCKER_CONTEXT=chainguard .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      image: family/kibana-ubuntu-2004
+      imageProject: elastic-images-qa
+      provider: gcp
+      machineType: n2-standard-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -81,7 +81,7 @@ steps:
           preemptible: true
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 2
+        parallelism: 3
         retry:
           automatic:
             - exit_status: '-1'
@@ -149,7 +149,7 @@ steps:
           preemptible: true
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 2
+        parallelism: 1
         retry:
           automatic:
             - exit_status: '-1'

.buildkite/pipelines/on_merge.yml

@@ -100,7 +100,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'
@@ -156,7 +156,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 1
     retry:
       automatic:
         - exit_status: '-1'
@@ -296,7 +296,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 3
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'
@@ -310,7 +310,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 7
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/build_project.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+    label: 'Build Project Image'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/pull_request/security_solution/entity_analytics.yml

@@ -7,7 +7,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/security_solution/explore.yml

@@ -7,7 +7,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 3
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/security_solution/investigations.yml

@@ -21,7 +21,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 7
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/pipelines/pull_request/security_solution/rule_management.yml

@@ -21,7 +21,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 1
     retry:
       automatic:
         - exit_status: '-1'

.buildkite/scripts/build_kibana.sh

@@ -32,6 +32,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --skip-docker-ubi \
   --skip-docker-fips \
   --skip-docker-ubuntu \
+  --skip-docker-chainguard \
   --skip-docker-serverless \
   --skip-docker-contexts
 

.buildkite/scripts/common/setup_job_env.sh

@@ -21,7 +21,7 @@ fi
 
   KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
   KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
-  if command -v docker &> /dev/null; then
+  if (command -v docker && docker version) &> /dev/null; then
     echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
   fi
 }

.buildkite/scripts/common/vault_fns.sh

@@ -65,3 +65,23 @@ vault_kv_set() {
 
   vault kv put "$VAULT_KV_PREFIX/$kv_path" "${fields[@]}"
 }
+
+function get_vault_role_id() {
+  if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
+  else
+    VAULT_ROLE_ID="$(vault_get kibana-buildkite-vault-credentials role-id)"
+  fi
+
+  echo "$VAULT_ROLE_ID"
+}
+
+function get_vault_secret_id() {
+    if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+      VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+    else
+      VAULT_SECRET_ID="$(vault_get kibana-buildkite-vault-credentials secret-id)"
+    fi
+
+    echo "$VAULT_SECRET_ID"
+}

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -151,6 +151,9 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fips.yml'));
     }
 
+    if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/build_project.yml'));
+    }
     if (
       GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
       GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||
@@ -159,10 +162,6 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
-      pipeline.push(getPipeline('.buildkite/pipelines/artifacts_container_image.yml'));
-    }
-
     if (
       (await doAnyChangesMatch([/.*stor(ies|y).*/])) ||
       GITHUB_PR_LABELS.includes('ci:build-storybooks')

.buildkite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh

@@ -34,6 +34,7 @@ node scripts/build \
   --docker-namespace="kibana-ci" \
   --docker-tag="$KIBANA_IMAGE_TAG" \
   --skip-docker-ubuntu \
+  --skip-docker-chainguard \
   --skip-docker-ubi \
   --skip-docker-cloud \
   --skip-docker-contexts \

.buildkite/scripts/pipelines/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh

@@ -10,7 +10,6 @@ fi
 
 
 source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 .buildkite/scripts/bootstrap.sh
 
 export JOB=kibana-defend-workflows-serverless-cypress

.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh

@@ -9,7 +9,6 @@ if [ -z "$1" ]
 fi
 
 source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 .buildkite/scripts/bootstrap.sh
 
 export JOB=kibana-security-solution-chrome

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -20,6 +20,9 @@ case $KIBANA_DOCKER_CONTEXT in
   default)
     DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
   ;;
+  chainguard)
+    DOCKER_CONTEXT_FILE="kibana-chainguard-$FULL_VERSION-docker-build-context.tar.gz"
+  ;;
   cloud)
     DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
   ;;

.buildkite/scripts/steps/artifacts/docker_image.sh

@@ -32,10 +32,10 @@ node scripts/build \
   --docker-namespace="kibana-ci" \
   --docker-tag="$KIBANA_IMAGE_TAG" \
   --skip-docker-ubuntu \
+  --skip-docker-chainguard \
   --skip-docker-ubi \
   --skip-docker-fips \
-  --skip-docker-cloud \
-  --skip-docker-contexts
+  --skip-docker-cloud
 
 echo "--- Tag images"
 docker rmi "$KIBANA_IMAGE"
@@ -101,8 +101,9 @@ ts-node "$(git rev-parse --show-toplevel)/.buildkite/scripts/steps/artifacts/val
 echo "--- Upload archives"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-linux-x86_64.tar.gz"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-linux-aarch64.tar.gz"
-buildkite-agent artifact upload "kibana-$BASE_VERSION-docker-image.tar.gz"
-buildkite-agent artifact upload "kibana-$BASE_VERSION-docker-image-aarch64.tar.gz"
+buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-image.tar.gz"
+buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-image-aarch64.tar.gz"
+buildkite-agent artifact upload "kibana-serverless-$BASE_VERSION-docker-build-context.tar.gz"
 buildkite-agent artifact upload "kibana-$BASE_VERSION-cdn-assets.tar.gz"
 buildkite-agent artifact upload "dependencies-$GIT_ABBREV_COMMIT.csv"
 

.buildkite/scripts/steps/artifacts/publish.sh

@@ -53,8 +53,8 @@ docker pull docker.elastic.co/infra/release-manager:latest
 
 echo "--- Publish artifacts"
 if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then
-  export VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-  export VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
+  export VAULT_ROLE_ID="$(get_vault_role_id)"
+  export VAULT_SECRET_ID="$(get_vault_secret_id)"
   export VAULT_ADDR="https://secrets.elastic.co:8200"
 
   download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"

.buildkite/scripts/steps/cloud/build_and_deploy.sh

@@ -43,6 +43,7 @@ else
     --skip-docker-ubi \
     --skip-docker-fips \
     --skip-docker-ubuntu \
+    --skip-docker-chainguard \
     --skip-docker-serverless \
     --skip-docker-contexts
 fi
@@ -80,13 +81,13 @@ if [ -z "${CLOUD_DEPLOYMENT_ID}" ] || [ "${CLOUD_DEPLOYMENT_ID}" = 'null' ]; the
   CLOUD_DEPLOYMENT_STATUS_MESSAGES=$(jq --slurp '[.[]|select(.resources == null)]' "$ECCTL_LOGS")
 
   echo "Writing to vault..."
-  VAULT_ROLE_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-role-id)"
-  VAULT_SECRET_ID="$(retry 5 15 gcloud secrets versions access latest --secret=kibana-buildkite-vault-secret-id)"
-  VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
-  retry 5 30 vault login -no-print "$VAULT_TOKEN"
 
   # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
   if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
+    VAULT_ROLE_ID="$(get_vault_role_id)"
+    VAULT_SECRET_ID="$(get_vault_secret_id)"
+    VAULT_TOKEN=$(retry 5 30 vault write -field=token auth/approle/login role_id="$VAULT_ROLE_ID" secret_id="$VAULT_SECRET_ID")
+    retry 5 30 vault login -no-print "$VAULT_TOKEN"
     vault_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
   else
     vault_kv_set "cloud-deploy/$CLOUD_DEPLOYMENT_NAME" username="$CLOUD_DEPLOYMENT_USERNAME" password="$CLOUD_DEPLOYMENT_PASSWORD"
@@ -122,16 +123,16 @@ else
   ecctl deployment update "$CLOUD_DEPLOYMENT_ID" --track --output json --file /tmp/deploy.json > "$ECCTL_LOGS"
 fi
 
-CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
-CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
-
 # TODO: remove after https://github.com/elastic/kibana-operations/issues/15 is done
 if [[ "$IS_LEGACY_VAULT_ADDR" == "true" ]]; then
   VAULT_READ_COMMAND="vault read $VAULT_PATH_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
 else
   VAULT_READ_COMMAND="vault kv get $VAULT_KV_PREFIX/cloud-deploy/$CLOUD_DEPLOYMENT_NAME"
 fi
 
+CLOUD_DEPLOYMENT_KIBANA_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.kibana[0].info.metadata.aliased_url')
+CLOUD_DEPLOYMENT_ELASTICSEARCH_URL=$(ecctl deployment show "$CLOUD_DEPLOYMENT_ID" | jq -r '.resources.elasticsearch[0].info.metadata.aliased_url')
+
 cat << EOF | buildkite-agent annotate --style "info" --context cloud
   ### Cloud Deployment
 

.buildkite/scripts/steps/fips/build.sh

@@ -23,6 +23,7 @@ node scripts/build \
     --docker-push \
     --skip-docker-ubi \
     --skip-docker-ubuntu \
+    --skip-docker-chainguard \
     --skip-docker-cloud \
     --skip-docker-serverless \
     --skip-docker-contexts

.buildkite/scripts/steps/fips/smoke_test.sh

@@ -36,9 +36,9 @@ for config in "${configs[@]}"; do
     echo "^^^ +++"
 
     if [[ "$failedConfigs" ]]; then
-      failedConfigs="${failedConfigs}"$'\n'"$config"
+      failedConfigs="${failedConfigs}"$'\n'"- ${config}"
     else
-      failedConfigs="$config"
+      failedConfigs="### Failed FTR Configs"$'\n'"- ${config}"
     fi
   fi
 done

.buildkite/scripts/steps/functional/defend_workflows.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-defend-workflows-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/plugins/security_solution
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci defend-workflows-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:run; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:run; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/defend_workflows_burn.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 .buildkite/scripts/bootstrap.sh
 node scripts/build_kibana_platform_plugins.js

.buildkite/scripts/steps/functional/defend_workflows_serverless.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-defend-workflows-serverless-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/plugins/security_solution
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci defend-workflows-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:serverless:run; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:serverless:run; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 .buildkite/scripts/bootstrap.sh
 node scripts/build_kibana_platform_plugins.js

.buildkite/scripts/steps/functional/fleet_cypress.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-fleet-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

.buildkite/scripts/steps/functional/osquery_cypress.sh

@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 # TODO: remove the line below to use build artifacts for tests.
 # in addition to remove the line, we will have to expose the kibana install dir into the downloaded build location