Merge remote-tracking branch 'upstream/main' into 158183-execution-ha…

…ndler-refactor

# Conflicts:
#	x-pack/plugins/alerting/server/task_runner/execution_handler.ts
#	x-pack/plugins/stack_connectors/common/experimental_features.ts
#	x-pack/plugins/triggers_actions_ui/tsconfig.json

.buildkite/ftr_configs.yml

@@ -15,7 +15,8 @@ disabled:
   - x-pack/test/fleet_api_integration/config.base.ts
   - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
   - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
-
+  - x-pack/test/security_solution_endpoint/config.base.ts
+  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -225,13 +226,8 @@ enabled:
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group1/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group4/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group5/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/group6/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/group10/config.ts
   - x-pack/test/detection_engine_api_integration/security_and_spaces/rule_execution_logic/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/prebuilt_rules/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/bundled_prebuilt_rules_package/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/large_prebuilt_rules_package/config.ts
-  - x-pack/test/detection_engine_api_integration/security_and_spaces/update_prebuilt_rules_package/config.ts
   - x-pack/test/disable_ems/config.ts
   - x-pack/test/encrypted_saved_objects_api_integration/config.ts
   - x-pack/test/examples/config.ts
@@ -386,8 +382,11 @@ enabled:
   - x-pack/test/security_functional/user_profiles.config.ts
   - x-pack/test/security_functional/expired_session.config.ts
   - x-pack/test/security_solution_endpoint_api_int/config.ts
+  - x-pack/test/security_solution_endpoint_api_int/serverless.config.ts
   - x-pack/test/security_solution_endpoint/endpoint.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.endpoint.config.ts
   - x-pack/test/security_solution_endpoint/integrations.config.ts
+  - x-pack/test/security_solution_endpoint/serverless.integrations.config.ts
   - x-pack/test/session_view/basic/config.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/config_basic.ts
   - x-pack/test/spaces_api_integration/security_and_spaces/copy_to_space_config_basic.ts
@@ -459,8 +458,15 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/actions/configs/ess.config.ts
-
-
-
-
-
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/alerts/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/entity_analytics/default_license/risk_engine/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/management/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/bundled_prebuilt_rules_package/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/large_prebuilt_rules_package/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/prebuilt_rules/update_prebuilt_rules_package/configs/ess.config.ts

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -273,7 +273,12 @@ export async function pickTestGroupRunOrder() {
           ]
         : []),
       // if we are running on a external job, like kibana-code-coverage-main, try finding times that are specific to that job
-      ...(!prNumber && pipelineSlug !== 'kibana-on-merge'
+      // kibana-elasticsearch-serverless-verify-and-promote is not necessarily run in commit order -
+      // using kibana-on-merge groups will provide a closer approximation, with a failure mode -
+      // of too many ftr groups instead of potential timeouts.
+      ...(!prNumber &&
+      pipelineSlug !== 'kibana-on-merge' &&
+      pipelineSlug !== 'kibana-elasticsearch-serverless-verify-and-promote'
         ? [
             {
               branch: ownBranch,

.buildkite/pipelines/artifacts.yml

@@ -71,6 +71,16 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: KIBANA_DOCKER_CONTEXT=ironbank .buildkite/scripts/steps/artifacts/docker_context.sh
+    label: 'Docker Context Verification'
+    agents:
+      queue: n2-2
+    timeout_in_minutes: 30
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
     soft_fail:

.buildkite/pipelines/es_serverless/emergency_relelease_branch_testing.yml

@@ -0,0 +1,17 @@
+# https://buildkite.com/elastic/kibana-serverless-emergency-release-branch-testing
+
+## Triggers the artifacts container image build for emergency releases
+agents:
+  queue: kibana-default
+
+notify:
+  - slack: "#kibana-mission-control"
+    if: "build.state == 'passed' || build.state == 'failed' || build.state == 'scheduled'"
+
+steps:
+  - trigger: "kibana-artifacts-container-image"
+    label: ":docker: Build Kibana Artifacts Container Image"
+    build:
+      branch: $BUILDKITE_BRANCH
+      commit: $BUILDKITE_COMMIT
+      message: Running PR build for $BUILDKITE_BRANCH

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -63,7 +63,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 4
+        parallelism: 6
         retry:
           automatic:
             - exit_status: '*'

.buildkite/pipelines/pull_request/deploy_project.yml

@@ -0,0 +1,11 @@
+steps:
+  - command: .buildkite/scripts/steps/serverless/build_and_deploy.sh
+    label: 'Build and Deploy Project'
+    agents:
+      queue: n2-16-spot
+    timeout_in_minutes: 60
+    soft_fail: true
+    retry:
+      automatic:
+        - exit_status: '-1'
+          limit: 3

.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml

@@ -5,7 +5,7 @@
 steps:
   - label: ":kibana: Kibana Serverless Tests for ${ENVIRONMENT}"
     trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests
-    soft_fail: true # Remove this before release or when tests stabilize
+    soft_fail: true # Remove when tests stabilize
     build:
       env:
         ENVIRONMENT: ${ENVIRONMENT}
@@ -16,7 +16,6 @@ steps:
   # TODO: Uncomment this code when the integration is ready.
   # - label: ":pipeline::female-detective::seedling: Trigger Security Solution quality gate script"
   #   trigger: security-serverless-quality-gate # https://buildkite.com/elastic/security-serverless-quality-gate
-  #   soft_fail: true # Remove this when tests are fixed
   #   build:
   #     env:
   #       ENVIRONMENT: ${ENVIRONMENT}
@@ -46,7 +45,7 @@ steps:
         env:
           USE_GROUP_LABEL: true
         agents:
-          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.4"
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"
 
   - group: "Observability"
     steps:
@@ -56,4 +55,4 @@ steps:
           NOTIFICATION_APPENDIX: "<!subteam^S060GJFKGGP> please execute your manual testing plan."
           USE_GROUP_LABEL: true
         agents:
-          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.4"
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -13,7 +13,7 @@ steps:
 
   - label: ":kibana: Kibana Serverless Tests for ${ENVIRONMENT}"
     trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests
-    soft_fail: true # Remove this before release or when tests stabilize
+    soft_fail: true # Remove when tests stabilize
     build:
       env:
         ENVIRONMENT: ${ENVIRONMENT}
@@ -23,7 +23,9 @@ steps:
 
   - wait: ~
 
-  - label: ":judge::seedling: Trigger Manual Tests Phase"
-    command: "make -C /agent trigger-manual-verification-phase"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.4"
+  - group: "Kibana Release Manager"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase"
+        command: "make -C /agent trigger-manual-verification-phase"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/security_solution/api_integration.yml

@@ -0,0 +1,67 @@
+steps:
+  - label: Running exception_workflows:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_workflows:qa:serverless
+    key: exception_workflows:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 2
+
+  - label: Running exception_operators_date_numeric_types:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_date_numeric_types:qa:serverless
+    key: exception_operators_date_numeric_types:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120  
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 2
+
+  - label: Running exception_operators_keyword_text_long:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_keyword_text_long:qa:serverless
+    key: exception_operators_keyword_text_long:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 2
+
+  - label: Running exception_operators_ips_text_array:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh exception_operators_ips_text_array:qa:serverless
+    key: exception_operators_ips_text_array:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running rule_creation:runner:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh rule_creation:qa:serverless
+    key: rule_creation:runner:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+
+  - label: Running actions:qa:serverless
+    command: .buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh actions:qa:serverless
+    key: actions:qa:serverless
+    agents:
+      queue: n2-4-spot
+    timeout_in_minutes: 120
+    retry:
+      automatic:
+        - exit_status: '1'
+          limit: 2
+

.buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -0,0 +1,36 @@
+steps:
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless
+    label: 'Serverless MKI QA Security Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
+    label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:investigations
+    label: 'Serverless MKI QA Investigations - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1

.buildkite/scripts/build_kibana.sh

@@ -14,6 +14,7 @@ is_pr_with_label "ci:build-docker-cross-compile" && BUILD_ARGS+=("--docker-cross
 is_pr_with_label "ci:build-os-packages" || BUILD_ARGS+=("--skip-os-packages")
 is_pr_with_label "ci:build-canvas-shareable-runtime" || BUILD_ARGS+=("--skip-canvas-shareable-runtime")
 is_pr_with_label "ci:build-docker-contexts" || BUILD_ARGS+=("--skip-docker-contexts")
+is_pr_with_label "ci:build-cdn-assets" || BUILD_ARGS+=("--skip-cdn-assets")
 
 echo "> node scripts/build" "${BUILD_ARGS[@]}"
 node scripts/build "${BUILD_ARGS[@]}"
@@ -24,6 +25,7 @@ if is_pr_with_label "ci:build-cloud-image"; then
   --skip-initialize \
   --skip-generic-folders \
   --skip-platform-folders \
+  --skip-cdn-assets \
   --skip-archives \
   --docker-images \
   --docker-tag-qualifier="$GIT_COMMIT" \

.buildkite/scripts/common/env.sh

@@ -28,6 +28,24 @@ export KIBANA_BASE_BRANCH="$KIBANA_PKG_BRANCH"
 KIBANA_PKG_VERSION="$(jq -r .version "$KIBANA_DIR/package.json")"
 export KIBANA_PKG_VERSION
 
+# Detects and exports the final target branch when using a merge queue
+if [[ "${BUILDKITE_BRANCH:-}" == "gh-readonly-queue"* ]]; then
+  # removes gh-readonly-queue/
+  BKBRANCH_WITHOUT_GH_MQ_PREFIX="${BUILDKITE_BRANCH#gh-readonly-queue/}"
+
+  # extracts target mqueue branch
+  MERGE_QUEUE_TARGET_BRANCH=${BKBRANCH_WITHOUT_GH_MQ_PREFIX%/*}
+else
+  MERGE_QUEUE_TARGET_BRANCH=""
+fi
+export MERGE_QUEUE_TARGET_BRANCH
+
+# Exports BUILDKITE_BRANCH_MERGE_QUEUE which will use the value from MERGE_QUEUE_TARGET_BRANCH if defined otherwise
+# will fallback to BUILDKITE_BRANCH.
+BUILDKITE_BRANCH_MERGE_QUEUE="${MERGE_QUEUE_TARGET_BRANCH:-${BUILDKITE_BRANCH:-}}"
+export BUILDKITE_BRANCH_MERGE_QUEUE
+
+
 BUILDKITE_AGENT_GCP_REGION=""
 if [[ "$(curl -is metadata.google.internal || true)" ]]; then
   # projects/1003139005402/zones/us-central1-a -> us-central1-a -> us-central1

.buildkite/scripts/lifecycle/post_command.sh

@@ -14,7 +14,9 @@ if [[ "$IS_TEST_EXECUTION_STEP" == "true" ]]; then
   buildkite-agent artifact upload 'target/kibana-coverage/functional/**/*'
   buildkite-agent artifact upload 'target/kibana-*'
   buildkite-agent artifact upload 'target/kibana-security-solution/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-security-solution/**/management/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-osquery/**/*.png'
+  buildkite-agent artifact upload 'target/kibana-osquery/**/*.mp4'
   buildkite-agent artifact upload 'target/kibana-fleet/**/*.png'
   buildkite-agent artifact upload 'target/test-metrics/*'
   buildkite-agent artifact upload 'target/test-suites-ci-plan.json'

.buildkite/scripts/lifecycle/pre_command.sh

@@ -115,6 +115,12 @@ export KIBANA_DOCKER_PASSWORD
 EC_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-cloud-deploy)"
 export EC_API_KEY
 
+PROJECT_API_KEY="$(retry 5 5 vault read -field=pr_deploy_api_key secret/kibana-issues/dev/kibana-ci-project-deploy)"
+export PROJECT_API_KEY
+
+PROJECT_API_DOMAIN="$(retry 5 5 vault read -field=pr_deploy_domain secret/kibana-issues/dev/kibana-ci-project-deploy)"
+export PROJECT_API_DOMAIN
+
 SYNTHETICS_SERVICE_USERNAME="$(retry 5 5 vault read -field=username secret/kibana-issues/dev/kibana-ci-synthetics-credentials)"
 export SYNTHETICS_SERVICE_USERNAME
 

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -136,6 +136,14 @@ const uploadPipeline = (pipelineContent: string | object) => {
       pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_cloud.yml'));
     }
 
+    if (
+      GITHUB_PR_LABELS.includes('ci:project-deploy-elasticsearch') ||
+      GITHUB_PR_LABELS.includes('ci:project-deploy-observability') ||
+      GITHUB_PR_LABELS.includes('ci:project-deploy-security')
+    ) {
+      pipeline.push(getPipeline('.buildkite/pipelines/pull_request/deploy_project.yml'));
+    }
+
     if (GITHUB_PR_LABELS.includes('ci:build-serverless-image')) {
       pipeline.push(getPipeline('.buildkite/pipelines/artifacts_container_image.yml'));
     }