Merge branch 'main' into discover-locator/grid

.buildkite/ftr_configs.yml

@@ -191,6 +191,7 @@ enabled:
   - x-pack/test/api_integration/apis/es/config.ts
   - x-pack/test/api_integration/apis/features/config.ts
   - x-pack/test/api_integration/apis/file_upload/config.ts
+  - x-pack/test/api_integration/apis/grok_debugger/config.ts
   - x-pack/test/api_integration/apis/kibana/config.ts
   - x-pack/test/api_integration/apis/lists/config.ts
   - x-pack/test/api_integration/apis/logs_ui/config.ts
@@ -362,6 +363,7 @@ enabled:
   - x-pack/test/search_sessions_integration/config.ts
   - x-pack/test/security_api_integration/anonymous_es_anonymous.config.ts
   - x-pack/test/security_api_integration/anonymous.config.ts
+  - x-pack/test/security_api_integration/api_keys.config.ts
   - x-pack/test/security_api_integration/audit.config.ts
   - x-pack/test/security_api_integration/http_bearer.config.ts
   - x-pack/test/security_api_integration/http_no_auth_providers.config.ts

.buildkite/pipeline-utils/buildkite/client.ts

@@ -7,24 +7,31 @@
  */
 
 import axios, { AxiosInstance } from 'axios';
-import { execSync } from 'child_process';
+import { execSync, ExecSyncOptions } from 'child_process';
 import { dump } from 'js-yaml';
 import { parseLinkHeader } from './parse_link_header';
 import { Artifact } from './types/artifact';
 import { Build, BuildStatus } from './types/build';
 import { Job, JobState } from './types/job';
 
+type ExecType =
+  | ((command: string, execOpts: ExecSyncOptions) => Buffer | null)
+  | ((command: string, execOpts: ExecSyncOptions) => string | null);
+
 export interface BuildkiteClientConfig {
   baseUrl?: string;
   token?: string;
+  exec?: ExecType;
 }
 
 export interface BuildkiteGroup {
   group: string;
   steps: BuildkiteStep[];
 }
 
-export interface BuildkiteStep {
+export type BuildkiteStep = BuildkiteCommandStep | BuildkiteInputStep;
+
+export interface BuildkiteCommandStep {
   command: string;
   label: string;
   parallelism?: number;
@@ -43,6 +50,50 @@ export interface BuildkiteStep {
   env?: { [key: string]: string };
 }
 
+interface BuildkiteInputTextField {
+  text: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+}
+
+interface BuildkiteInputSelectField {
+  select: string;
+  key: string;
+  hint?: string;
+  required?: boolean;
+  default?: string;
+  multiple?: boolean;
+  options: Array<{
+    label: string;
+    value: string;
+  }>;
+}
+
+export interface BuildkiteInputStep {
+  input: string;
+  prompt?: string;
+  fields: Array<BuildkiteInputTextField | BuildkiteInputSelectField>;
+  if?: string;
+  allow_dependency_failure?: boolean;
+  branches?: string;
+  parallelism?: number;
+  agents?: {
+    queue: string;
+  };
+  timeout_in_minutes?: number;
+  key?: string;
+  depends_on?: string | string[];
+  retry?: {
+    automatic: Array<{
+      exit_status: string;
+      limit: number;
+    }>;
+  };
+  env?: { [key: string]: string };
+}
+
 export interface BuildkiteTriggerBuildParams {
   commit: string;
   branch: string;
@@ -61,6 +112,7 @@ export interface BuildkiteTriggerBuildParams {
 
 export class BuildkiteClient {
   http: AxiosInstance;
+  exec: ExecType;
 
   constructor(config: BuildkiteClientConfig = {}) {
     const BUILDKITE_BASE_URL =
@@ -78,6 +130,8 @@ export class BuildkiteClient {
       },
     });
 
+    this.exec = config.exec ?? execSync;
+
     // this.agentHttp = axios.create({
     //   baseURL: BUILDKITE_AGENT_BASE_URL,
     //   headers: {
@@ -97,6 +151,32 @@ export class BuildkiteClient {
     return resp.data as Build;
   };
 
+  getBuildsAfterDate = async (
+    pipelineSlug: string,
+    date: string,
+    numberOfBuilds: number
+  ): Promise<Build[]> => {
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?created_from=${date}&per_page=${numberOfBuilds}`
+    );
+    return response.data as Build[];
+  };
+
+  getBuildForCommit = async (pipelineSlug: string, commit: string): Promise<Build | null> => {
+    if (commit.length !== 40) {
+      throw new Error(`Invalid commit hash: ${commit}, this endpoint works with full SHAs only`);
+    }
+
+    const response = await this.http.get(
+      `v2/organizations/elastic/pipelines/${pipelineSlug}/builds?commit=${commit}`
+    );
+    const builds = response.data as Build[];
+    if (builds.length === 0) {
+      return null;
+    }
+    return builds[0];
+  };
+
   getCurrentBuild = (includeRetriedJobs = false) => {
     if (!process.env.BUILDKITE_PIPELINE_SLUG || !process.env.BUILDKITE_BUILD_NUMBER) {
       throw new Error(
@@ -235,31 +315,52 @@ export class BuildkiteClient {
   };
 
   setMetadata = (key: string, value: string) => {
-    execSync(`buildkite-agent meta-data set '${key}'`, {
+    this.exec(`buildkite-agent meta-data set '${key}'`, {
       input: value,
       stdio: ['pipe', 'inherit', 'inherit'],
     });
   };
 
+  getMetadata(key: string, defaultValue: string | null = null): string | null {
+    try {
+      const stdout = this.exec(`buildkite-agent meta-data get '${key}'`, {
+        stdio: ['pipe'],
+      });
+      return stdout?.toString().trim() || defaultValue;
+    } catch (e) {
+      if (e.message.includes('404 Not Found')) {
+        return defaultValue;
+      } else {
+        throw e;
+      }
+    }
+  }
+
   setAnnotation = (
     context: string,
     style: 'info' | 'success' | 'warning' | 'error',
-    value: string
+    value: string,
+    append: boolean = false
   ) => {
-    execSync(`buildkite-agent annotate --context '${context}' --style '${style}'`, {
-      input: value,
-      stdio: ['pipe', 'inherit', 'inherit'],
-    });
+    this.exec(
+      `buildkite-agent annotate --context '${context}' --style '${style}' ${
+        append ? '--append' : ''
+      }`,
+      {
+        input: value,
+        stdio: ['pipe', 'inherit', 'inherit'],
+      }
+    );
   };
 
   uploadArtifacts = (pattern: string) => {
-    execSync(`buildkite-agent artifact upload '${pattern}'`, {
+    this.exec(`buildkite-agent artifact upload '${pattern}'`, {
       stdio: ['ignore', 'inherit', 'inherit'],
     });
   };
 
   uploadSteps = (steps: Array<BuildkiteStep | BuildkiteGroup>) => {
-    execSync(`buildkite-agent pipeline upload`, {
+    this.exec(`buildkite-agent pipeline upload`, {
       input: dump({ steps }),
       stdio: ['pipe', 'inherit', 'inherit'],
     });

.buildkite/pipeline-utils/github/github.ts

@@ -91,3 +91,7 @@ export const doAnyChangesMatch = async (
 
   return anyFilesMatchRequired;
 };
+
+export function getGithubClient() {
+  return github;
+}

.buildkite/pipeline-utils/index.ts

@@ -10,3 +10,4 @@ export * from './buildkite';
 export * as CiStats from './ci-stats';
 export * from './github';
 export * as TestFailures from './test-failures';
+export * from './utils';

.buildkite/pipeline-utils/utils.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { execSync } from 'child_process';
+
+const getKibanaDir = (() => {
+  let kibanaDir: string | undefined;
+  return () => {
+    if (!kibanaDir) {
+      kibanaDir = execSync('git rev-parse --show-toplevel', { encoding: 'utf-8' })
+        .toString()
+        .trim();
+    }
+
+    return kibanaDir;
+  };
+})();
+
+export { getKibanaDir };

.buildkite/pipelines/es_serverless/verify_es_serverless_image.yml

@@ -56,30 +56,20 @@ steps:
             - exit_status: '*'
               limit: 1
 
-      - command: .buildkite/scripts/steps/functional/security_cypress_exec.sh 
-        env:
-          TARGET: cypress:run:serverless
-          ROOT_DIR: x-pack/test/security_solution_cypress
-          JOB_TITLE: kibana-serverless-security-cypress
-          MSG: "--- Security Serverless Cypress Tests"
+      - command: .buildkite/scripts/steps/functional/security_serverless.sh
         label: 'Serverless Security Cypress Tests'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
-        parallelism: 6
+        parallelism: 2
         retry:
           automatic:
             - exit_status: '*'
               limit: 1
 
-      - command: .buildkite/scripts/steps/functional/security_cypress_exec.sh
-        env:
-          TARGET: cypress:explore:run:serverless
-          ROOT_DIR: x-pack/test/security_solution_cypress
-          JOB_TITLE: kibana-security-solution-chrome
-          MSG: "--- Explore - Security Solution Cypress Tests"
+      - command: .buildkite/scripts/steps/functional/security_serverless_explore.sh
         label: 'Serverless Explore - Security Solution Cypress Tests'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
@@ -92,12 +82,7 @@ steps:
             - exit_status: '*'
               limit: 1
 
-      - command: .buildkite/scripts/steps/functional/security_cypress_exec.sh
-        env:
-          TARGET: cypress:investigations:run:serverless
-          ROOT_DIR: x-pack/test/security_solution_cypress
-          JOB_TITLE: kibana-security-solution-chrome
-          MSG: "--- Investigations Cypress Tests on Serverless"
+      - command: .buildkite/scripts/steps/functional/security_serverless_investigations.sh
         label: 'Serverless Investigations - Security Solution Cypress Tests'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
@@ -110,12 +95,7 @@ steps:
             - exit_status: '*'
               limit: 1
 
-      - command: .buildkite/scripts/steps/functional/security_cypress_exec.sh
-        env:
-          TARGET: cypress:rule_management:run:serverless
-          ROOT_DIR: x-pack/test/security_solution_cypress
-          JOB_TITLE: kibana-security-solution-chrome
-          MSG: "--- Rule Management Cypress Tests on Serverless"
+      - command: .buildkite/scripts/steps/functional/security_serverless_rule_management.sh
         label: 'Serverless Rule Management - Security Solution Cypress Tests'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
@@ -128,30 +108,46 @@ steps:
             - exit_status: '*'
               limit: 1
 
-      - command: .buildkite/scripts/steps/functional/security_cypress_exec.sh
-        env:
-          TARGET: cypress:rule_management:prebuilt_rules:run:serverless
-          ROOT_DIR: x-pack/test/security_solution_cypress
-          JOB_TITLE: kibana-security-solution-chrome
-          MSG: "--- Rule Management - Prebuilt Rules - Cypress Tests on Serverless"
+      - command: .buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh
         label: 'Serverless Rule Management - Prebuilt Rules - Security Solution Cypress Tests'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents:
           queue: n2-4-spot
         depends_on: build
         timeout_in_minutes: 60
+        parallelism: 4
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine.sh
+        label: 'Serverless Detection Engine - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
         parallelism: 6
         retry:
           automatic:
             - exit_status: '*'
               limit: 1
 
-      - command: .buildkite/scripts/steps/functional/security_cypress_exec.sh
-        env:
-          TARGET: cypress:dw:serverless:run
-          ROOT_DIR: x-pack/plugins/security_solution
-          JOB_TITLE: kibana-defend-workflows-serverless-cypress
-          MSG: "--- Defend Workflows Cypress tests on Serverless"
+      - command: .buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh
+        label: 'Serverless Detection Engine - Exceptions - Security Solution Cypress Tests'
+        if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
+        agents:
+          queue: n2-4-spot
+        depends_on: build
+        timeout_in_minutes: 60
+        parallelism: 6
+        retry:
+          automatic:
+            - exit_status: '*'
+              limit: 1
+
+      - command: .buildkite/scripts/steps/functional/defend_workflows_serverless.sh
         label: 'Defend Workflows Cypress Tests on Serverless'
         if: "build.env('SKIP_CYPRESS') != '1' && build.env('SKIP_CYPRESS') != 'true'"
         agents: