Merge branch 'main' of github.com:elastic/kibana into aiassistant/get…

…_dataviews

.buildkite/ftr_configs.yml

@@ -13,6 +13,9 @@ disabled:
   - x-pack/test/functional_with_es_ssl/config.base.ts
   - x-pack/test/api_integration/config.ts
   - x-pack/test/fleet_api_integration/config.base.ts
+  - x-pack/test/security_solution_api_integration/config/ess/config.base.ts
+  - x-pack/test/security_solution_api_integration/config/serverless/config.base.ts
+
 
   # QA suites that are run out-of-band
   - x-pack/test/stack_functional_integration/configs/config.stack_functional_integration_base.js
@@ -338,7 +341,6 @@ enabled:
   - x-pack/test/monitoring_api_integration/config.ts
   - x-pack/test/observability_api_integration/basic/config.ts
   - x-pack/test/observability_api_integration/trial/config.ts
-  - x-pack/test/observability_api_integration/apis/config.ts
   - x-pack/test/observability_functional/with_rac_write.config.ts
   - x-pack/test/observability_onboarding_api_integration/basic/config.ts
   - x-pack/test/observability_onboarding_api_integration/cloud/config.ts
@@ -449,3 +451,9 @@ enabled:
   - x-pack/performance/journeys/apm_service_inventory.ts
   - x-pack/test/custom_branding/config.ts
   - x-pack/test/profiling_api_integration/cloud/config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/detections_response/ess.config.ts
+
+
+
+

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -226,6 +226,10 @@ export async function pickTestGroupRunOrder() {
           .filter(Boolean)
       : ['build'];
 
+  const FTR_EXTRA_ARGS: Record<string, string> = process.env.FTR_EXTRA_ARGS
+    ? { FTR_EXTRA_ARGS: process.env.FTR_EXTRA_ARGS }
+    : {};
+
   const { defaultQueue, ftrConfigsByQueue } = getEnabledFtrConfigs(FTR_CONFIG_PATTERNS);
 
   const ftrConfigsIncluded = LIMIT_CONFIG_TYPE.includes('functional');
@@ -464,6 +468,7 @@ export async function pickTestGroupRunOrder() {
                   },
                   env: {
                     FTR_CONFIG_GROUP_KEY: key,
+                    ...FTR_EXTRA_ARGS,
                   },
                   retry: {
                     automatic: [

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -1,4 +1,14 @@
 # https://buildkite.com/elastic/kibana-elasticsearch-serverless-verify-and-promote/
+
+### Parameters for this job:
+# PUBLISH_DOCKER_TAG: if set to 1/true, passing runs will promote the tested ES Serverless tag to latest-verified.
+# PUBLISH_MANIFEST: if set to 1/true, passing runs will upload the manifest attesting what (kibana + es) combination was used in the test
+# SKIP_VERIFICATION: if set to 1/true, it will skip running all tests
+# SKIP_CYPRESS: if set to 1/true, it will skip running the cypress tests
+# FTR_EXTRA_ARGS: a string argument, if passed, it will be forwarded verbatim to the FTR run script
+# ES_SERVERLESS_IMAGE: the tag for the docker image to test, in the form of docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:$TAG
+# BUILDKITE_COMMIT: the commit hash of the kibana branch to test
+
 agents:
   queue: kibana-default
 
@@ -24,7 +34,6 @@ steps:
           queue: n2-16-spot
         key: build
         depends_on: pre-build
-        if: "build.env('KIBANA_BUILD_ID') == null || build.env('KIBANA_BUILD_ID') == ''"
         timeout_in_minutes: 60
         retry:
           automatic:
@@ -35,10 +44,12 @@ steps:
         command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh
         agents:
           queue: kibana-default
+        depends_on: build
         timeout_in_minutes: 10
         env:
           FTR_CONFIGS_SCRIPT: 'TEST_ES_SERVERLESS_IMAGE=$ES_SERVERLESS_IMAGE .buildkite/scripts/steps/test/ftr_configs.sh'
           FTR_CONFIG_PATTERNS: '**/test_serverless/**'
+          FTR_EXTRA_ARGS: '$FTR_EXTRA_ARGS'
           LIMIT_CONFIG_TYPE: 'functional'
         retry:
           automatic:
@@ -47,6 +58,7 @@ steps:
 
       - command: .buildkite/scripts/steps/functional/security_serverless.sh
         label: 'Serverless Security Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build
@@ -59,6 +71,7 @@ steps:
 
       - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
         label: 'Serverless Explore - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build
@@ -71,6 +84,7 @@ steps:
 
       - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
         label: 'Serverless Investigations - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build

.buildkite/pipelines/flaky_tests/groups.json

@@ -24,6 +24,14 @@
       "key": "cypress/security_serverless_explore",
       "name": "[Serverless] Security Solution Explore - Cypress"
     },
+    {
+      "key": "cypress/defend_workflows",
+      "name": "Security Solution Defend Workflows - Cypress"
+    },
+    {
+      "key": "cypress/defend_workflows_serverless",
+      "name": "[Serverless] Security Solution Defend Workflows - Cypress"
+    },
     {
       "key": "cypress/osquery_cypress",
       "name": "Osquery - Cypress"

.buildkite/pipelines/on_merge.yml

@@ -270,6 +270,13 @@ steps:
         - exit_status: '-1'
           limit: 3
 
+  - command: .buildkite/scripts/steps/archive_so_migration_snapshot.sh target/plugin_so_types_snapshot.json
+    label: 'Extract Saved Object migration plugin types'
+    agents:
+      queue: n2-4-spot
+    artifact_paths:
+      "target/plugin_so_types_snapshot.json"
+
   - wait: ~
     continue_on_failure: true
 

.buildkite/pipelines/pipeline.kibana-serverless-release.yaml

@@ -2,7 +2,7 @@ steps:
   - label: ":releasethekraken: Release kibana"
     # https://regex101.com/r/tY52jo/1
     if: build.tag =~ /^deploy@\d+\$/
-    trigger: gpctl-promote-with-e2e-tests
+    trigger: gpctl-promote
     build:
       env:
         SERVICE_COMMIT_HASH: "${BUILDKITE_COMMIT:0:12}"

.buildkite/pipelines/pull_request/base.yml

@@ -63,7 +63,7 @@ steps:
       queue: n2-4-spot
     depends_on: build
     timeout_in_minutes: 60
-    parallelism: 2
+    parallelism: 4
     retry:
       automatic:
         - exit_status: '*'

.buildkite/pipelines/quality-gates/pipeline.tests-production-canary.yaml

@@ -0,0 +1,25 @@
+# These pipeline steps constitute the quality gate for your service within the production-canary environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":pipeline::kibana::seedling: Trigger SLO check"
+    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-canary.yaml)"
+      env:
+        TARGET_ENV: production-canary
+        CHECK_SLO: true
+        CHECK_SLO_TAG: kibana
+    soft_fail: true
+
+  - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests"
+    trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-canary.yaml)"
+
+  - label: ":cookie: 24h bake time before continuing promotion"
+    command: "sleep 86400"

.buildkite/pipelines/quality-gates/pipeline.tests-production-noncanary.yaml

@@ -0,0 +1,22 @@
+# These pipeline steps constitute the quality gate for your service within the production-noncanary environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":pipeline::kibana::seedling: Trigger SLO check"
+    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-noncanary.yaml)"
+      env:
+        TARGET_ENV: production-noncanary
+        CHECK_SLO: true
+        CHECK_SLO_TAG: kibana
+    soft_fail: true
+
+  - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests"
+    trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production-noncanary.yaml)"

.buildkite/pipelines/quality-gates/pipeline.tests-production.yaml

@@ -2,19 +2,25 @@
 # Incorporate any necessary additional logic to validate the service's integrity.
 # A failure in this pipeline build will prevent further progression to the subsequent stage.
 
+# DEPRECATION NOTICE:
+# PRODUCTION WILL SOON BE SPLIT INTO "CANARY" AND "NONCANARY" AND THIS FILE WILL BE DELETED.
+# ENSURE ANY CHANGE MADE TO THIS FILE IS REFLECTED IN THOSE FILES AS WELL.
+
 steps:
+  - label: ":pipeline::kibana::seedling: Trigger SLO check"
+    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
+      env:
+        TARGET_ENV: production
+        CHECK_SLO: true
+        CHECK_SLO_TAG: kibana
+    soft_fail: true
+
   - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests"
     trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests
     build:
       env:
         REGION_ID: aws-us-east-1
         NAME_PREFIX: ci_test_kibana-promotion_
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
-
-  - wait: ~
-
-  - label: ":judge::seedling: Trigger Manual Tests Phase"
-    command: "make -C /agent trigger-manual-verification-phase"
-    if: build.branch == "main"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.2"

.buildkite/pipelines/quality-gates/pipeline.tests-qa.yaml

@@ -3,18 +3,6 @@
 # A failure in this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
-  - label: ":pipeline::kibana::seedling: Trigger SLO check"
-    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
-    build:
-      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"
-      env:
-        TARGET_ENV: qa
-        CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
-        CHECK_SLO_WAITING_PERIOD: 10m
-        CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
-    soft_fail: true
-
   - label: ":pipeline::kibana::seedling: Trigger Kibana Serverless Tests for ${ENVIRONMENT}"
     trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests
     soft_fail: true # Remove this before release or when tests stabilize
@@ -30,6 +18,7 @@ steps:
     steps:
       - label: ":pipeline::female-detective::seedling: Trigger Security Solution quality gate script"
         command: .buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh
+        soft_fail: true # Remove this when tests are fixed
 
   - label: ":pipeline::ship::seedling: Trigger Fleet serverless smoke tests for ${ENVIRONMENT}"
     trigger: fleet-smoke-tests # https://buildkite.com/elastic/fleet-smoke-tests

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -3,16 +3,6 @@
 # A failure in this pipeline build will prevent further progression to the subsequent stage.
 
 steps:
-  - label: ":pipeline::kibana::seedling: Trigger SLO check"
-    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
-    build:
-      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
-      env:
-        TARGET_ENV: staging
-        CHECK_SLO: true
-        CHECK_SLO_TAG: kibana
-    soft_fail: true
-
   - label: ":pipeline::rocket::seedling: Trigger control-plane e2e tests"
     trigger: "ess-k8s-staging-e2e-tests" # https://buildkite.com/elastic/ess-k8s-staging-e2e-tests
     build:

.buildkite/scripts/pipelines/security_solution_quality_gate/pipeline.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
-
 set -euo pipefail
 
-echo "In the entrypoint for the quality gate"
+source .buildkite/scripts/common/util.sh
+source .buildkite/scripts/steps/functional/common_cypress.sh
+.buildkite/scripts/bootstrap.sh
+
+export JOB=kibana-security-solution-chrome
+
+buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
+
+echo "--- Serverless Security Second Quality Gate"
+cd x-pack/test/security_solution_cypress
+set +e
+
+VAULT_DEC_KEY=$(vault read -field=key secret/kibana-issues/dev/security-solution-qg-enc-key)
+ENV_PWD=$(echo $TEST_ENV_PWD | openssl aes-256-cbc -d -a -pass pass:$VAULT_DEC_KEY)
+
+CYPRESS_ELASTICSEARCH_URL=$TEST_ENV_ES_URL CYPRESS_BASE_URL=$TEST_ENV_KB_URL CYPRESS_ELASTICSEARCH_USERNAME=$TEST_ENV_USERNAME CYPRESS_ELASTICSEARCH_PASSWORD=$ENV_PWD CYPRESS_KIBANA_URL=$CYPRESS_BASE_URL yarn cypress:run:qa:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/archive_so_migration_snapshot.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+.buildkite/scripts/bootstrap.sh
+
+SO_MIGRATIONS_SNAPSHOT_FOLDER=kibana-so-types-snapshots
+SNAPSHOT_FILE_PATH="${1:-target/plugin_so_types_snapshot.json}"
+
+echo "--- Creating snapshot of Saved Object migration info"
+node scripts/snapshot_plugin_types --outputPath "$SNAPSHOT_FILE_PATH"
+
+echo "--- Uploading as ${BUILDKITE_COMMIT}.json"
+SNAPSHOT_PATH="${SO_MIGRATIONS_SNAPSHOT_FOLDER}/${BUILDKITE_COMMIT}.json"
+gsutil cp "$SNAPSHOT_FILE_PATH" "gs://$SNAPSHOT_PATH"
+
+buildkite-agent annotate --context so_migration_snapshot --style success \
+  'Saved Object type snapshot is available at <a href="https://storage.cloud.google.com/'"$SNAPSHOT_PATH"'">'"$SNAPSHOT_PATH"'</a>'
+
+echo "Success!"

.buildkite/scripts/steps/es_serverless/promote_es_serverless_image.sh

@@ -19,7 +19,12 @@ else
   SOURCE_IMAGE="$BASE_ES_SERVERLESS_REPO:$SOURCE_IMAGE_OR_TAG"
 fi
 
-echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+if [[ "${PUBLISH_DOCKER_TAG:-}" =~ ^(1|true)$ ]]; then
+  echo "--- Promoting ${SOURCE_IMAGE_OR_TAG} to ':latest-verified'"
+else
+  echo "--- Skipping ES Serverless image because PUBLISH_DOCKER_TAG is not set"
+  exit 0
+fi
 
 echo "Re-tagging $SOURCE_IMAGE -> $TARGET_IMAGE"
 
@@ -63,7 +68,7 @@ echo "Image push to $TARGET_IMAGE successful."
 echo "Promotion successful! Henceforth, thou shall be named Sir $TARGET_IMAGE"
 
 MANIFEST_UPLOAD_PATH="Skipped"
-if [[ "${UPLOAD_MANIFEST:-}" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
+if [[ "${PUBLISH_MANIFEST:-}" =~ ^(1|true)$ && "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
   echo "--- Uploading latest-verified manifest to GCS"
   cat << EOT >> $MANIFEST_FILE_NAME
 {
@@ -84,7 +89,7 @@ EOT
   gsutil acl ch -u AllUsers:R "gs://$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME"
   MANIFEST_UPLOAD_PATH="<a href=\"https://storage.googleapis.com/$ES_SERVERLESS_BUCKET/$MANIFEST_FILE_NAME\">$MANIFEST_FILE_NAME</a>"
 
-elif [[ "${UPLOAD_MANIFEST:-}" =~ ^(1|true)$ ]]; then
+elif [[ "${PUBLISH_MANIFEST:-}" =~ ^(1|true)$ ]]; then
   echo "--- Skipping upload of latest-verified manifest to GCS, ES Serverless build tag is not pointing to a hash"
 elif [[ "$SOURCE_IMAGE_OR_TAG" =~ ^git-[0-9a-fA-F]{12}$ ]]; then
   echo "--- Skipping upload of latest-verified manifest to GCS, flag was not provided"

.buildkite/scripts/steps/test/ftr_configs.sh

@@ -11,6 +11,9 @@ if [ "$FTR_CONFIG_GROUP_KEY" == "" ] && [ "$BUILDKITE_PARALLEL_JOB" == "" ]; the
   exit 1
 fi
 
+EXTRA_ARGS=${FTR_EXTRA_ARGS:-}
+test -z "$EXTRA_ARGS" || buildkite-agent meta-data set "ftr-extra-args" "$EXTRA_ARGS"
+
 export JOB="$FTR_CONFIG_GROUP_KEY"
 
 FAILED_CONFIGS_KEY="${BUILDKITE_STEP_ID}${FTR_CONFIG_GROUP_KEY}"
@@ -49,15 +52,18 @@ while read -r config; do
     continue;
   fi
 
-  echo "--- $ node scripts/functional_tests --bail --config $config"
+  FULL_COMMAND="node scripts/functional_tests --bail --config $config $EXTRA_ARGS"
+  echo "--- $ $FULL_COMMAND"
+
   start=$(date +%s)
 
   # prevent non-zero exit code from breaking the loop
   set +e;
   node ./scripts/functional_tests \
     --bail \
     --kibana-install-dir "$KIBANA_BUILD_LOCATION" \
-    --config="$config"
+    --config="$config" \
+    "$EXTRA_ARGS"
   lastCode=$?
   set -e;
 

.eslintrc.js

@@ -606,6 +606,8 @@ module.exports = {
         'x-pack/test_serverless/**/config*.ts',
         'x-pack/test_serverless/*/test_suites/**/*',
         'x-pack/test/profiling_api_integration/**/*.ts',
+        'x-pack/test/security_solution_api_integration/*/test_suites/**/*',
+        'x-pack/test/security_solution_api_integration/**/config*.ts',
       ],
       rules: {
         'import/no-default-export': 'off',