Merge remote-tracking branch 'kibana/main' into siem-ea-8355

elastic · May 13, 2024 · 47dfd61 · 47dfd61
2 parents fa26277 + 551e123
commit 47dfd61
Show file tree

Hide file tree

Showing 2,398 changed files with 49,580 additions and 141,764 deletions.
diff --git a/.buildkite/ftr_configs.yml b/.buildkite/ftr_configs.yml
@@ -215,7 +215,6 @@ enabled:
   - x-pack/test/api_integration/apis/search/config.ts
   - x-pack/test/api_integration/apis/searchprofiler/config.ts
   - x-pack/test/api_integration/apis/security/config.ts
-  - x-pack/test/api_integration/apis/security_solution/config.ts
   - x-pack/test/api_integration/apis/spaces/config.ts
   - x-pack/test/api_integration/apis/stats/config.ts
   - x-pack/test/api_integration/apis/status/config.ts
@@ -321,6 +320,7 @@ enabled:
   - x-pack/test/functional/apps/saved_objects_management/config.ts
   - x-pack/test/functional/apps/saved_query_management/config.ts
   - x-pack/test/functional/apps/security/config.ts
+  - x-pack/test/functional/apps/slo/embeddables/config.ts
   - x-pack/test/functional/apps/snapshot_restore/config.ts
   - x-pack/test/functional/apps/spaces/config.ts
   - x-pack/test/functional/apps/spaces/in_solution_navigation/config.ts
@@ -550,3 +550,17 @@ enabled:
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/exception_lists_items/trial_license_complete_tier/configs/serverless.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/ess.config.ts
   - x-pack/test/security_solution_api_integration/test_suites/lists_and_exception_lists/lists_items/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/hosts/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/network/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/network/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/users/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/users/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/overview/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/explore/overview/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/saved_objects/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/investigation/timeline/trial_license_complete_tier/configs/serverless.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/ess.config.ts
+  - x-pack/test/security_solution_api_integration/test_suites/sources/indices/trial_license_complete_tier/configs/serverless.config.ts
diff --git a/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml b/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml
@@ -20,8 +20,9 @@ spec:
     spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
-        ES_SERVERLESS_IMAGE: latest
+        ES_SERVERLESS_IMAGE: docker.elastic.co/elasticsearch-ci/elasticsearch-serverless:latest
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        REPORT_FAILED_TESTS_TO_GITHUB: 'true'
       allow_rebuilds: true
       branch_configuration: main
       default_branch: main

diff --git a/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml b/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml
@@ -128,6 +128,7 @@ spec:
       env:
         SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts'
         ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true'
+        REPORT_FAILED_TESTS_TO_GITHUB: 'true'
       allow_rebuilds: true
       branch_configuration: main 8.14 7.17
       default_branch: main

diff --git a/.buildkite/pipeline-utils/buildkite/client.ts b/.buildkite/pipeline-utils/buildkite/client.ts
@@ -29,12 +29,19 @@ export interface BuildkiteGroup {
   steps: BuildkiteStep[];
 }
 
-export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep | BuildkiteTriggerStep;
+export type BuildkiteStep =
+  | BuildkiteCommandStep
+  | BuildkiteInputStep
+  | BuildkiteTriggerStep
+  | BuildkiteWaitStep;
 
 export interface BuildkiteCommandStep {
   command: string;
   label: string;
   parallelism?: number;
+  concurrency?: number;
+  concurrency_group?: string;
+  concurrency_method?: 'eager' | 'ordered';
   agents:
     | {
         queue: string;
@@ -49,14 +56,15 @@ export interface BuildkiteCommandStep {
       };
   timeout_in_minutes?: number;
   key?: string;
+  cancel_on_build_failing?: boolean;
   depends_on?: string | string[];
   retry?: {
     automatic: Array<{
       exit_status: string;
       limit: number;
     }>;
   };
-  env?: { [key: string]: string };
+  env?: { [key: string]: string | number };
 }
 
 interface BuildkiteInputTextField {
@@ -100,7 +108,7 @@ export interface BuildkiteInputStep {
       limit: number;
     }>;
   };
-  env?: { [key: string]: string };
+  env?: { [key: string]: string | number };
 }
 
 export interface BuildkiteTriggerStep {
@@ -138,6 +146,14 @@ export interface BuildkiteTriggerBuildParams {
   pull_request_repository?: string;
 }
 
+export interface BuildkiteWaitStep {
+  wait: string;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  continue_on_failure?: boolean;
+  branches?: string;
+}
+
 export class BuildkiteClient {
   http: AxiosInstance;
   exec: ExecType;

diff --git a/.buildkite/pipelines/build_api_docs.yml b/.buildkite/pipelines/build_api_docs.yml
@@ -6,7 +6,7 @@ steps:
     key: type_check
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
       preemptible: true
@@ -16,7 +16,7 @@ steps:
     label: 'Build API Docs'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
       preemptible: true
@@ -33,7 +33,7 @@ steps:
     label: 'Publish API Docs'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
       preemptible: true

diff --git a/.buildkite/pipelines/code_coverage/daily.yml b/.buildkite/pipelines/code_coverage/daily.yml
@@ -4,7 +4,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
 
@@ -14,7 +14,7 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
     timeout_in_minutes: 10
@@ -28,7 +28,7 @@ steps:
     label: 'Merge and Ingest'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
       preemptible: true

diff --git a/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml b/.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml
@@ -10,15 +10,22 @@
 
 agents:
   image: family/kibana-ubuntu-2004
-  imageProject: elastic-images-qa
+  imageProject: elastic-images-prod
   provider: gcp
   machineType: n2-standard-2
 
 steps:
   - label: "Annotate runtime parameters"
     command: |
-      buildkite-agent annotate --context es-serverless-image --style info "ES Serverless image: $ES_SERVERLESS_IMAGE"
       buildkite-agent annotate --context kibana-commit --style info "Kibana build hash: $BUILDKITE_BRANCH / $BUILDKITE_COMMIT"
+      cat << EOF | buildkite-agent annotate --context es-serverless-image --style info
+        ES Serverless image: \`$ES_SERVERLESS_IMAGE\`
+
+        To run this locally:
+        \`\`\`
+        node scripts/es serverless --image $ES_SERVERLESS_IMAGE
+        \`\`\`
+      EOF
 
   - group: "(:kibana: x :elastic:) Trigger Kibana Serverless suite"
     if: "build.env('SKIP_VERIFICATION') != '1' && build.env('SKIP_VERIFICATION') != 'true'"
@@ -29,15 +36,15 @@ steps:
         timeout_in_minutes: 10
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-2
 
       - label: "Build Kibana Distribution and Plugins"
         command: .buildkite/scripts/steps/build_kibana.sh
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-16
         key: build
@@ -68,7 +75,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -85,7 +92,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -102,7 +109,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -119,7 +126,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -136,7 +143,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -153,7 +160,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -170,7 +177,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -187,7 +194,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true
@@ -204,7 +211,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           enableNestedVirtualization: true
           localSsds: 1
@@ -223,7 +230,7 @@ steps:
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           image: family/kibana-ubuntu-2004
-          imageProject: elastic-images-qa
+          imageProject: elastic-images-prod
           provider: gcp
           machineType: n2-standard-4
           preemptible: true

diff --git a/.buildkite/pipelines/es_snapshots/build.yml b/.buildkite/pipelines/es_snapshots/build.yml
@@ -4,7 +4,7 @@ steps:
     timeout_in_minutes: 30
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme

diff --git a/.buildkite/pipelines/es_snapshots/promote.yml b/.buildkite/pipelines/es_snapshots/promote.yml
@@ -12,6 +12,6 @@ steps:
     command: .buildkite/scripts/steps/es_snapshots/promote.sh
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
diff --git a/.buildkite/pipelines/es_snapshots/verify.yml b/.buildkite/pipelines/es_snapshots/verify.yml
@@ -16,7 +16,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
 
@@ -26,7 +26,7 @@ steps:
     label: Build Kibana Distribution and Plugins
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-16
       preemptible: true
@@ -42,7 +42,7 @@ steps:
     label: 'Pick Test Group Run Order'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
     timeout_in_minutes: 10
@@ -61,7 +61,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
     depends_on:
@@ -76,6 +76,6 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
diff --git a/.buildkite/pipelines/esql_grammar_sync.yml b/.buildkite/pipelines/esql_grammar_sync.yml
@@ -4,7 +4,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
       preemptible: true
diff --git a/.buildkite/pipelines/fips.yml b/.buildkite/pipelines/fips.yml
@@ -5,7 +5,7 @@ steps:
     label: Build Kibana Distribution and Plugins
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-16
       preemptible: true
@@ -23,7 +23,7 @@ steps:
     label: "Smoke testing for FIPS"
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       enableNestedVirtualization: true
       localSsds: 1