Merge branch 'main' into test_discoverability

elastic · May 15, 2024 · 9b56a74 · 9b56a74
2 parents 6f39ead + 42b2a5a
commit 9b56a74
Show file tree

Hide file tree

Showing 1,088 changed files with 8,461 additions and 4,942 deletions.
diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
@@ -92,6 +92,16 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: KIBANA_DOCKER_CONTEXT=chainguard .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:

diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
@@ -81,7 +81,7 @@ steps:
           preemptible: true
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 2
+        parallelism: 3
         retry:
           automatic:
             - exit_status: '-1'
@@ -149,7 +149,7 @@ steps:
           preemptible: true
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 2
+        parallelism: 1
         retry:
           automatic:
             - exit_status: '-1'

diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml
@@ -100,7 +100,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'
@@ -156,7 +156,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 1
     retry:
       automatic:
         - exit_status: '-1'
@@ -296,7 +296,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 3
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'
@@ -310,7 +310,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 7
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/pipelines/pull_request/build_project.yml b/.buildkite/pipelines/pull_request/build_project.yml
@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/artifacts/docker_image.sh
+    label: 'Build Project Image'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3
diff --git a/.buildkite/pipelines/pull_request/security_solution/entity_analytics.yml b/.buildkite/pipelines/pull_request/security_solution/entity_analytics.yml
@@ -7,7 +7,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 3
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/pipelines/pull_request/security_solution/explore.yml b/.buildkite/pipelines/pull_request/security_solution/explore.yml
@@ -7,7 +7,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 3
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/pipelines/pull_request/security_solution/investigations.yml b/.buildkite/pipelines/pull_request/security_solution/investigations.yml
@@ -21,7 +21,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 7
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/pipelines/pull_request/security_solution/rule_management.yml b/.buildkite/pipelines/pull_request/security_solution/rule_management.yml
@@ -21,7 +21,7 @@ steps:
       - build
       - quick_checks
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 1
     retry:
       automatic:
         - exit_status: '-1'

diff --git a/.buildkite/scripts/build_kibana.sh b/.buildkite/scripts/build_kibana.sh
@@ -32,6 +32,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --skip-docker-ubi \
   --skip-docker-fips \
   --skip-docker-ubuntu \
+  --skip-docker-chainguard \
   --skip-docker-serverless \
   --skip-docker-contexts
 

diff --git a/.buildkite/scripts/common/setup_job_env.sh b/.buildkite/scripts/common/setup_job_env.sh
@@ -21,7 +21,7 @@ fi
 
   KIBANA_DOCKER_USERNAME="$(vault_get container-registry username)"
   KIBANA_DOCKER_PASSWORD="$(vault_get container-registry password)"
-  if command -v docker &> /dev/null; then
+  if (command -v docker && docker version) &> /dev/null; then
     echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
   fi
 }

diff --git a/.buildkite/scripts/pipelines/pull_request/pipeline.ts b/.buildkite/scripts/pipelines/pull_request/pipeline.ts
@@ -151,6 +151,9 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/fips.yml'));
     }
 
+    if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/build_project.yml'));
+    }
     if (
       GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
       GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||
@@ -159,10 +162,6 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));
     }
 
-    if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
-      pipeline.push(getPipeline('.buildkite/pipelines/artifacts_container_image.yml'));
-    }
-
     if (
       (await doAnyChangesMatch([/.*stor(ies|y).*/])) ||
       GITHUB_PR_LABELS.includes('ci:build-storybooks')

diff --git a/...ite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh b/...ite/scripts/pipelines/security_solution_quality_gate/create_periodic_test_docker_image.sh
@@ -34,6 +34,7 @@ node scripts/build \
   --docker-namespace="kibana-ci" \
   --docker-tag="$KIBANA_IMAGE_TAG" \
   --skip-docker-ubuntu \
+  --skip-docker-chainguard \
   --skip-docker-ubi \
   --skip-docker-cloud \
   --skip-docker-contexts \

diff --git a/...es/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh b/...es/security_solution_quality_gate/edr_workflows/mki_security_solution_defend_workflows.sh
@@ -10,7 +10,6 @@ fi
 
 
 source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 .buildkite/scripts/bootstrap.sh
 
 export JOB=kibana-defend-workflows-serverless-cypress

diff --git a/...security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh b/...security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh
@@ -9,7 +9,6 @@ if [ -z "$1" ]
 fi
 
 source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 .buildkite/scripts/bootstrap.sh
 
 export JOB=kibana-security-solution-chrome

diff --git a/.buildkite/scripts/steps/artifacts/docker_context.sh b/.buildkite/scripts/steps/artifacts/docker_context.sh
@@ -20,6 +20,9 @@ case $KIBANA_DOCKER_CONTEXT in
   default)
     DOCKER_CONTEXT_FILE="kibana-$FULL_VERSION-docker-build-context.tar.gz"
   ;;
+  chainguard)
+    DOCKER_CONTEXT_FILE="kibana-chainguard-$FULL_VERSION-docker-build-context.tar.gz"
+  ;;
   cloud)
     DOCKER_CONTEXT_FILE="kibana-cloud-$FULL_VERSION-docker-build-context.tar.gz"
   ;;

diff --git a/.buildkite/scripts/steps/artifacts/docker_image.sh b/.buildkite/scripts/steps/artifacts/docker_image.sh
@@ -32,6 +32,7 @@ node scripts/build \
   --docker-namespace="kibana-ci" \
   --docker-tag="$KIBANA_IMAGE_TAG" \
   --skip-docker-ubuntu \
+  --skip-docker-chainguard \
   --skip-docker-ubi \
   --skip-docker-fips \
   --skip-docker-cloud \

diff --git a/.buildkite/scripts/steps/cloud/build_and_deploy.sh b/.buildkite/scripts/steps/cloud/build_and_deploy.sh
@@ -43,6 +43,7 @@ else
     --skip-docker-ubi \
     --skip-docker-fips \
     --skip-docker-ubuntu \
+    --skip-docker-chainguard \
     --skip-docker-serverless \
     --skip-docker-contexts
 fi

diff --git a/.buildkite/scripts/steps/fips/build.sh b/.buildkite/scripts/steps/fips/build.sh
@@ -23,6 +23,7 @@ node scripts/build \
     --docker-push \
     --skip-docker-ubi \
     --skip-docker-ubuntu \
+    --skip-docker-chainguard \
     --skip-docker-cloud \
     --skip-docker-serverless \
     --skip-docker-contexts

diff --git a/.buildkite/scripts/steps/functional/common_cypress.sh b/.buildkite/scripts/steps/functional/common_cypress.sh
diff --git a/.buildkite/scripts/steps/functional/defend_workflows.sh b/.buildkite/scripts/steps/functional/defend_workflows.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-defend-workflows-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/plugins/security_solution
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci defend-workflows-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:run; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:run; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/defend_workflows_burn.sh b/.buildkite/scripts/steps/functional/defend_workflows_burn.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 .buildkite/scripts/bootstrap.sh
 node scripts/build_kibana_platform_plugins.js

diff --git a/.buildkite/scripts/steps/functional/defend_workflows_serverless.sh b/.buildkite/scripts/steps/functional/defend_workflows_serverless.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-defend-workflows-serverless-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/plugins/security_solution
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci defend-workflows-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:serverless:run; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:serverless:run; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh b/.buildkite/scripts/steps/functional/defend_workflows_serverless_burn.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 .buildkite/scripts/bootstrap.sh
 node scripts/build_kibana_platform_plugins.js

diff --git a/.buildkite/scripts/steps/functional/fleet_cypress.sh b/.buildkite/scripts/steps/functional/fleet_cypress.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-fleet-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

diff --git a/.buildkite/scripts/steps/functional/osquery_cypress.sh b/.buildkite/scripts/steps/functional/osquery_cypress.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 # TODO: remove the line below to use build artifacts for tests.
 # in addition to remove the line, we will have to expose the kibana install dir into the downloaded build location

diff --git a/.buildkite/scripts/steps/functional/osquery_cypress_burn.sh b/.buildkite/scripts/steps/functional/osquery_cypress_burn.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/common/util.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 .buildkite/scripts/bootstrap.sh
 node scripts/build_kibana_platform_plugins.js
@@ -14,4 +13,4 @@ buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" 'fals
 
 echo "--- Osquery Cypress tests, burning changed specs (Chrome)"
 
-yarn --cwd x-pack/plugins/osquery cypress:changed-specs-only
+yarn --cwd x-pack/plugins/osquery cypress:changed-specs-only
diff --git a/.buildkite/scripts/steps/functional/response_ops.sh b/.buildkite/scripts/steps/functional/response_ops.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

diff --git a/.buildkite/scripts/steps/functional/response_ops_cases.sh b/.buildkite/scripts/steps/functional/response_ops_cases.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

diff --git a/.buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh b/.buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/test/security_solution_cypress
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:ai_assistant:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:ai_assistant:run:serverless; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/security_serverless_burn.sh b/.buildkite/scripts/steps/functional/security_serverless_burn.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}

diff --git a/.buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh b/.buildkite/scripts/steps/functional/security_serverless_defend_workflows.sh
@@ -3,11 +3,10 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-serverless-security-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
 echo "--- Security Defend Workflows Serverless Cypress"
 
-yarn --cwd x-pack/test_serverless/functional/test_suites/security/cypress cypress:run
+yarn --cwd x-pack/test_serverless/functional/test_suites/security/cypress cypress:run
diff --git a/.buildkite/scripts/steps/functional/security_serverless_detection_engine.sh b/.buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/test/security_solution_cypress
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:run:serverless; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh b/.buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-security-solution-chrome
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/test/security_solution_cypress
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:exceptions:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:exceptions:run:serverless; status=$?; yarn junit:merge || :; exit $status
diff --git a/.buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh b/.buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
@@ -3,7 +3,6 @@
 set -euo pipefail
 
 source .buildkite/scripts/steps/functional/common.sh
-source .buildkite/scripts/steps/functional/common_cypress.sh
 
 export JOB=kibana-serverless-security-cypress
 export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
@@ -15,4 +14,4 @@ cd x-pack/test/security_solution_cypress
 set +e
 BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
 
-BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:entity_analytics:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:entity_analytics:run:serverless; status=$?; yarn junit:merge || :; exit $status