Merge branch 'main' into ml-add-e5-config

.buildkite/ftr_configs.yml

@@ -158,7 +158,9 @@ enabled:
   - test/server_integration/http/ssl_with_p12/config.js
   - test/server_integration/http/ssl/config.js
   - test/ui_capabilities/newsfeed_err/config.ts
-  - x-pack/test/accessibility/config.ts
+  - x-pack/test/accessibility/apps/group1/config.ts
+  - x-pack/test/accessibility/apps/group2/config.ts
+  - x-pack/test/accessibility/apps/group3/config.ts
   - x-pack/test/localization/config.ja_jp.ts
   - x-pack/test/localization/config.fr_fr.ts
   - x-pack/test/localization/config.zh_cn.ts
@@ -331,7 +333,6 @@ enabled:
   - x-pack/test/kubernetes_security/basic/config.ts
   - x-pack/test/licensing_plugin/config.public.ts
   - x-pack/test/licensing_plugin/config.ts
-  - x-pack/test/lists_api_integration/security_and_spaces/config.ts
   - x-pack/test/monitoring_api_integration/config.ts
   - x-pack/test/observability_api_integration/basic/config.ts
   - x-pack/test/observability_api_integration/trial/config.ts
@@ -490,4 +491,9 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/telemetry/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/detections_response/basic_essentials_license/detection_engine/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/exception_lists_items/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/default_license/lists_items/configs/ess.config.ts
+
 

.buildkite/pipelines/artifacts.yml

@@ -61,7 +61,7 @@ steps:
         - exit_status: '*'
           limit: 1
 
-  - command: KIBANA_DOCKER_CONTEXT=ubi9 .buildkite/scripts/steps/artifacts/docker_context.sh
+  - command: KIBANA_DOCKER_CONTEXT=ubi .buildkite/scripts/steps/artifacts/docker_context.sh
     label: 'Docker Context Verification'
     agents:
       queue: n2-2

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -95,6 +95,32 @@ steps:
             - exit_status: '*'
               limit: 1
 
+      - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
+        label: 'Serverless Rule Management - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 8
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
+        label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 6
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
       - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
         label: 'Defend Workflows Cypress Tests on Serverless'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"

.buildkite/pipelines/on_merge.yml

@@ -115,6 +115,54 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
+    label: 'Serverless Rule Management - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
+    label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh
+    label: 'Rule Management - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh
+    label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:

.buildkite/pipelines/pull_request/base.yml

@@ -93,6 +93,30 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
+    label: 'Serverless Rule Management - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
+    label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:
@@ -117,6 +141,30 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh
+    label: 'Rule Management - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh
+    label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
     label: 'Investigations - Security Solution Cypress Tests'
     agents:

.buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -30,6 +30,30 @@ steps:
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management
+    label: 'Serverless MKI QA Rule Management - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:rule_management:prebuilt_rules
+    label: 'Serverless MKI QA Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '*'

.buildkite/scripts/packer_cache.sh

@@ -14,5 +14,9 @@ for version in $(cat versions.json | jq -r '.versions[].version'); do
   node scripts/es snapshot --download-only --base-path "$ES_CACHE_DIR" --version "$version"
 done
 
+for version in $(cat versions.json | jq -r '.versions[].version'); do
+  node x-pack/plugins/security_solution/scripts/endpoint/agent_downloader --version "$version"
+done
+
 echo "--- Cloning repos for docs build"
 node scripts/validate_next_docs --clone-only

.buildkite/scripts/steps/artifacts/docker_context.sh

@@ -26,8 +26,9 @@ case $KIBANA_DOCKER_CONTEXT in
   ubi8)
     DOCKER_CONTEXT_FILE="kibana-ubi8-$FULL_VERSION-docker-build-context.tar.gz"
   ;;
-  ubi9)
-    DOCKER_CONTEXT_FILE="kibana-ubi9-$FULL_VERSION-docker-build-context.tar.gz"
+  ubi)
+    # Currently ubi9.  After ubi8 we're moving to a version agnostic filename
+    DOCKER_CONTEXT_FILE="kibana-ubi-$FULL_VERSION-docker-build-context.tar.gz"
   ;;
   ironbank)
     DOCKER_CONTEXT_FILE="kibana-ironbank-$FULL_VERSION-docker-build-context.tar.gz"

.buildkite/scripts/steps/cloud/purge_deployments.ts

@@ -23,12 +23,11 @@ const DAY_IN_SECONDS = 60 * 60 * 24;
 for (const deployment of prDeployments) {
   try {
     const prNumber = deployment.name.match(/^kibana-pr-([0-9]+)$/)[1];
-    const prJson = execSync(`gh pr view '${prNumber}' --json state,labels,commits`).toString();
+    const prJson = execSync(`gh pr view '${prNumber}' --json state,labels,updatedAt`).toString();
     const pullRequest = JSON.parse(prJson);
     const prOpen = pullRequest.state === 'OPEN';
 
-    const lastCommit = pullRequest.commits.slice(-1)[0];
-    const lastCommitTimestamp = new Date(lastCommit.committedDate).getTime() / 1000;
+    const lastCommitTimestamp = new Date(pullRequest.updatedAt).getTime() / 1000;
 
     const persistDeployment = Boolean(
       pullRequest.labels.filter((label: any) => label.name === 'ci:cloud-persist-deployment').length

.buildkite/scripts/steps/cloud/purge_projects.ts

@@ -66,12 +66,11 @@ async function purgeProjects() {
     const NOW = new Date().getTime() / 1000;
     const DAY_IN_SECONDS = 60 * 60 * 24;
     const prJson = execSync(
-      `gh pr view '${project.prNumber}' --json state,labels,commits`
+      `gh pr view '${project.prNumber}' --json state,labels,updatedAt`
     ).toString();
     const pullRequest = JSON.parse(prJson);
     const prOpen = pullRequest.state === 'OPEN';
-    const lastCommit = pullRequest.commits.slice(-1)[0];
-    const lastCommitTimestamp = new Date(lastCommit.committedDate).getTime() / 1000;
+    const lastCommitTimestamp = new Date(pullRequest.updatedAt).getTime() / 1000;
 
     const persistDeployment = Boolean(
       pullRequest.labels.filter((label: any) => label.name === 'ci:project-persist-deployment')

.buildkite/scripts/steps/functional/security_serverless_rule_management.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Rule Management Cypress Tests on Serverless"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:rule_management:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Rule Management - Prebuilt Rules - Cypress Tests on Serverless"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:rule_management:prebuilt_rules:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_rule_management.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Rule Management - Security Solution Cypress Tests"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:rule_management:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/steps/functional/common.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+
+export JOB=kibana-security-solution-chrome
+export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
+
+echo "--- Rule Management - Prebuilt Rules - Security Solution Cypress Tests"
+
+cd x-pack/test/security_solution_cypress
+
+set +e
+yarn cypress:rule_management:prebuilt_rules:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/serverless/build_and_deploy.sh

@@ -105,3 +105,5 @@ EOF
 is_pr_with_label "ci:project-deploy-elasticsearch" && deploy "elasticsearch"
 is_pr_with_label "ci:project-deploy-observability" && deploy "observability"
 is_pr_with_label "ci:project-deploy-security" && deploy "security"
+
+exit 0;