Merge branch 'main' into max-con-reporting

.buildkite/ftr_configs.yml

@@ -191,6 +191,7 @@ enabled:
   - x-pack/test/api_integration/apis/es/config.ts
   - x-pack/test/api_integration/apis/features/config.ts
   - x-pack/test/api_integration/apis/file_upload/config.ts
+  - x-pack/test/api_integration/apis/grok_debugger/config.ts
   - x-pack/test/api_integration/apis/kibana/config.ts
   - x-pack/test/api_integration/apis/lists/config.ts
   - x-pack/test/api_integration/apis/logs_ui/config.ts
@@ -362,6 +363,7 @@ enabled:
   - x-pack/test/search_sessions_integration/config.ts
   - x-pack/test/security_api_integration/anonymous_es_anonymous.config.ts
   - x-pack/test/security_api_integration/anonymous.config.ts
+  - x-pack/test/security_api_integration/api_keys.config.ts
   - x-pack/test/security_api_integration/audit.config.ts
   - x-pack/test/security_api_integration/http_bearer.config.ts
   - x-pack/test/security_api_integration/http_no_auth_providers.config.ts

.buildkite/pipeline-utils/buildkite/client.ts

@@ -339,12 +339,18 @@ export class BuildkiteClient {
   setAnnotation = (
     context: string,
     style: 'info' | 'success' | 'warning' | 'error',
-    value: string
+    value: string,
+    append: boolean = false
   ) => {
-    this.exec(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
-      input: value,
-      stdio: ['pipe', 'inherit', 'inherit'],
-    });
+    this.exec(
+      `buildkite-agent annotate --context '${context}' --style '${style}' ${
+        append ? '--append' : ''
+      }`,
+      {
+        input: value,
+        stdio: ['pipe', 'inherit', 'inherit'],
+      }
+    );
   };
 
   uploadArtifacts = (pattern: string) => {

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -63,7 +63,7 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 6
+        parallelism: 2
         retry:
           automatic:
             - exit_status: '*'
@@ -115,6 +115,32 @@ steps:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
+        parallelism: 4
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
+        label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 6
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
+        label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
         parallelism: 6
         retry:
           automatic:

.buildkite/pipelines/flaky_tests/groups.json

@@ -41,6 +41,23 @@
       "key": "cypress/security_serverless_rule_management_prebuilt_rules",
       "name": "[Serverless] Security Solution Rule Management - Prebuilt Rules - Cypress"
     },
+    {
+      "key": "cypress/security_solution_detection_engine",
+      "name": "Security Solution Detection Engine - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_detection_engine",
+      "name": "[Serverless] Security Solution Detection Engine - Cypress"
+    },
+
+    {
+      "key": "cypress/security_solution_detection_engine_exceptions",
+      "name": "Security Solution Detection Engine - Exceptions - Cypress"
+    },
+    {
+      "key": "cypress/security_serverless_detection_engine_exceptions",
+      "name": "[Serverless] Security Solution Detection Engine - Exceptions - Cypress"
+    },
     {
       "key": "cypress/defend_workflows",
       "name": "Security Solution Defend Workflows - Cypress"

.buildkite/pipelines/on_merge.yml

@@ -85,7 +85,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
@@ -133,7 +133,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'
@@ -163,13 +163,61 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
+    label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
+    label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh
+    label: 'Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
+    label: 'Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_solution.sh
     label: 'Security Solution Cypress Tests'
     agents:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 16
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/pull_request/base.yml

@@ -63,7 +63,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 6
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
@@ -111,6 +111,30 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
+    parallelism: 4
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
+    label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
+    label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
     parallelism: 6
     retry:
       automatic:
@@ -123,7 +147,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 16
+    parallelism: 8
     retry:
       automatic:
         - exit_status: '*'
@@ -164,6 +188,30 @@ steps:
       automatic:
         - exit_status: '*'
           limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh
+    label: 'Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh
+    label: 'Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    depends_on: build
+    timeout_in_minutes: 60
+    parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
 
   - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh
     label: 'Investigations - Security Solution Cypress Tests'

.buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -54,6 +54,30 @@ steps:
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
     parallelism: 6
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine
+    label: 'Serverless MKI QA Detection Engine - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 8
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
+  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:detection_engine:exceptions
+    label: 'Serverless MKI QA Detection Engine - Exceptions - Security Solution Cypress Tests'
+    agents:
+      queue: n2-4-spot
+    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
+    timeout_in_minutes: 300
+    parallelism: 6
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/serverless_deployment/create_deployment_tag.yml

@@ -8,9 +8,11 @@ steps:
     commands:
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state initialize
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state collect_commits
-      - ts-node .buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts 25
+      - ts-node .buildkite/scripts/serverless/create_deploy_tag/list_commit_candidates.ts 50
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_selection
     key: select_commit
+    env:
+      AUTO_SELECT_COMMIT: $AUTO_SELECT_COMMIT
 
   - wait: ~
 
@@ -21,6 +23,8 @@ steps:
       - ts-node .buildkite/scripts/serverless/create_deploy_tag/release_wizard_messaging.ts --state wait_for_confirmation
     key: collect_data
     depends_on: select_commit
+    env:
+      AUTO_SELECT_COMMIT: $AUTO_SELECT_COMMIT
 
   - wait: ~
 

.buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.sh

@@ -8,9 +8,14 @@ set -euo pipefail
 echo "--- Collecting commit info"
 ts-node .buildkite/scripts/serverless/create_deploy_tag/collect_commit_info.ts
 
-cat << EOF | buildkite-agent pipeline upload
+if [[ "$AUTO_SELECT_COMMIT" == "true" || "$AUTO_SELECT_COMMIT" == "1" ]]; then
+  echo "--- Auto promoting to RC, skipping confirmation"
+else
+  echo "--- Uploading confirmation step"
+  cat << EOF | buildkite-agent pipeline upload
   steps:
     - block: "Confirm deployment"
       prompt: "Are you sure you want to deploy to production? (dry run: ${DRY_RUN:-false})"
       depends_on: collect_data
 EOF
+fi

.buildkite/scripts/serverless/create_deploy_tag/info_sections/so_snapshot_comparison.ts

@@ -52,7 +52,7 @@ export function makeSOComparisonBlockHtml(comparisonResult: {
   if (comparisonResult.hasChanges) {
     return `<div>
 <h4>Plugin Saved Object migration changes: *yes, ${comparisonResult.changed.length} plugin(s)*</h4>
-<div>Changed plugins: ${comparisonResult.changed.join(', ')}</div>
+<div>Changed plugins: <strong>${comparisonResult.changed.join(', ')}</strong></div>
 <i>Find detailed info in the archived artifacts, or run the command yourself: </i>
 <div><pre>${comparisonResult.command}</pre></div>
 </div>`;