Merge branch 'main' into siem-explore-issue-124499-fix

elastic · Mar 8, 2022 · ac09fe2 · ac09fe2
2 parents ad872f4 + 5ad355e
commit ac09fe2
Show file tree

Hide file tree

Showing 453 changed files with 6,943 additions and 6,828 deletions.
diff --git a/.buildkite/pipelines/hourly.yml b/.buildkite/pipelines/hourly.yml
@@ -19,7 +19,7 @@ steps:
     label: 'Default CI Group'
     parallelism: 27
     agents:
-      queue: n2-4-spot
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 250
     key: default-cigroup
@@ -31,7 +31,7 @@ steps:
   - command: CI_GROUP=Docker .buildkite/scripts/steps/functional/xpack_cigroup.sh
     label: 'Docker CI Group'
     agents:
-      queue: n2-4-spot
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     key: default-cigroup-docker
@@ -44,7 +44,7 @@ steps:
     label: 'OSS CI Group'
     parallelism: 11
     agents:
-      queue: n2-4-spot
+      queue: ci-group-4d
     depends_on: build
     timeout_in_minutes: 120
     key: oss-cigroup
@@ -56,49 +56,29 @@ steps:
   - command: .buildkite/scripts/steps/functional/oss_accessibility.sh
     label: 'OSS Accessibility Tests'
     agents:
-      queue: n2-4-spot
+      queue: ci-group-4d
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
-        - exit_status: '1'
+        - exit_status: '*'
           limit: 1
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '130'
-          limit: 3
-        - exit_status: '137'
-          limit: 3
-        - exit_status: '143'
-          limit: 3
-        - exit_status: '255'
-          limit: 3
 
   - command: .buildkite/scripts/steps/functional/xpack_accessibility.sh
     label: 'Default Accessibility Tests'
     agents:
-      queue: n2-4-spot
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
-        - exit_status: '1'
+        - exit_status: '*'
           limit: 1
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '130'
-          limit: 3
-        - exit_status: '137'
-          limit: 3
-        - exit_status: '143'
-          limit: 3
-        - exit_status: '255'
-          limit: 3
 
   - command: .buildkite/scripts/steps/functional/oss_firefox.sh
     label: 'OSS Firefox Tests'
     agents:
-      queue: n2-4-spot
+      queue: ci-group-4d
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -109,28 +89,18 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_firefox.sh
     label: 'Default Firefox Tests'
     agents:
-      queue: n2-4-spot
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
-        - exit_status: '1'
+        - exit_status: '*'
           limit: 1
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '130'
-          limit: 3
-        - exit_status: '137'
-          limit: 3
-        - exit_status: '143'
-          limit: 3
-        - exit_status: '255'
-          limit: 3
 
   - command: .buildkite/scripts/steps/functional/oss_misc.sh
     label: 'OSS Misc Functional Tests'
     agents:
-      queue: n2-4-spot
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
@@ -141,23 +111,13 @@ steps:
   - command: .buildkite/scripts/steps/functional/xpack_saved_object_field_metrics.sh
     label: 'Saved Object Field Metrics'
     agents:
-      queue: n2-4-spot
+      queue: n2-4
     depends_on: build
     timeout_in_minutes: 120
     retry:
       automatic:
-        - exit_status: '1'
+        - exit_status: '*'
           limit: 1
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '130'
-          limit: 3
-        - exit_status: '137'
-          limit: 3
-        - exit_status: '143'
-          limit: 3
-        - exit_status: '255'
-          limit: 3
 
   - command: .buildkite/scripts/steps/test/jest.sh
     label: 'Jest Tests'
@@ -178,23 +138,9 @@ steps:
   - command: .buildkite/scripts/steps/test/api_integration.sh
     label: 'API Integration Tests'
     agents:
-      queue: n2-4-spot
+      queue: n2-2
     timeout_in_minutes: 120
     key: api-integration
-    retry:
-      automatic:
-        - exit_status: '1'
-          limit: 1
-        - exit_status: '-1'
-          limit: 3
-        - exit_status: '130'
-          limit: 3
-        - exit_status: '137'
-          limit: 3
-        - exit_status: '143'
-          limit: 3
-        - exit_status: '255'
-          limit: 3
 
   - command: .buildkite/scripts/steps/lint.sh
     label: 'Linting'

diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -425,8 +425,8 @@ x-pack/plugins/session_view @elastic/awp-platform
 # Security Asset Management
 /x-pack/plugins/osquery @elastic/security-asset-management
 
-# Cloud Posture Security
-/x-pack/plugins/cloud_security_posture/ @elastic/cloud-posture-security
+# Cloud Security Posture
+/x-pack/plugins/cloud_security_posture/ @elastic/cloud-security-posture-control-plane
 
 # Design (at the bottom for specificity of SASS files)
 **/*.scss  @elastic/kibana-design

diff --git a/docs/development/core/server/kibana-plugin-core-server.elasticsearchclient.md b/docs/development/core/server/kibana-plugin-core-server.elasticsearchclient.md
@@ -9,5 +9,5 @@ Client used to query the elasticsearch cluster.
 <b>Signature:</b>
 
 ```typescript
-export declare type ElasticsearchClient = Omit<Client, 'connectionPool' | 'serializer' | 'extend' | 'child' | 'close' | 'diagnostic'>;
+export declare type ElasticsearchClient = Omit<Client, 'connectionPool' | 'serializer' | 'extend' | 'close' | 'diagnostic'>;
 ```
diff --git a/packages/kbn-doc-links/src/get_doc_links.ts b/packages/kbn-doc-links/src/get_doc_links.ts
@@ -196,6 +196,7 @@ export const getDocLinks = ({ kibanaBranch }: GetDocLinkOptions): DocLinks => {
       std_dev: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-extendedstats-aggregation.html`,
       sum: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-sum-aggregation.html`,
       top_hits: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-top-hits-aggregation.html`,
+      top_metrics: `${ELASTICSEARCH_DOCS}search-aggregations-metrics-top-metrics.html`,
     },
     runtimeFields: {
       overview: `${ELASTICSEARCH_DOCS}runtime.html`,

diff --git a/packages/kbn-test/BUILD.bazel b/packages/kbn-test/BUILD.bazel
@@ -53,6 +53,7 @@ RUNTIME_DEPS = [
     "@npm//execa",
     "@npm//exit-hook",
     "@npm//form-data",
+    "@npm//get-port",
     "@npm//getopts",
     "@npm//globby",
     "@npm//he",
@@ -90,6 +91,7 @@ TYPES_DEPS = [
   "@npm//del",
   "@npm//exit-hook",
   "@npm//form-data",
+  "@npm//get-port",
   "@npm//getopts",
   "@npm//jest",
   "@npm//jest-cli",

diff --git a/packages/kbn-test/src/es/es_client_for_testing.ts b/packages/kbn-test/src/es/es_client_for_testing.ts
@@ -27,6 +27,22 @@ export interface EsClientForTestingOptions extends Omit<ClientOptions, 'node' |
   isCloud?: boolean;
 }
 
+export function createRemoteEsClientForFtrConfig(
+  config: Config,
+  overrides?: Omit<EsClientForTestingOptions, 'esUrl'>
+) {
+  const ccsConfig = config.get('esTestCluster.ccs');
+  if (!ccsConfig) {
+    throw new Error('FTR config is missing esTestCluster.ccs');
+  }
+
+  return createEsClientForTesting({
+    esUrl: ccsConfig.remoteClusterUrl,
+    requestTimeout: config.get('timeouts.esRequestTimeout'),
+    ...overrides,
+  });
+}
+
 export function createEsClientForFtrConfig(
   config: Config,
   overrides?: Omit<EsClientForTestingOptions, 'esUrl'>

diff --git a/packages/kbn-test/src/es/index.ts b/packages/kbn-test/src/es/index.ts
@@ -9,5 +9,9 @@
 export { createTestEsCluster } from './test_es_cluster';
 export type { CreateTestEsClusterOptions, EsTestCluster, ICluster } from './test_es_cluster';
 export { esTestConfig } from './es_test_config';
-export { createEsClientForTesting, createEsClientForFtrConfig } from './es_client_for_testing';
+export {
+  createEsClientForTesting,
+  createEsClientForFtrConfig,
+  createRemoteEsClientForFtrConfig,
+} from './es_client_for_testing';
 export type { EsClientForTestingOptions } from './es_client_for_testing';
diff --git a/packages/kbn-test/src/es/test_es_cluster.ts b/packages/kbn-test/src/es/test_es_cluster.ts
@@ -136,7 +136,15 @@ export interface CreateTestEsClusterOptions {
    * }
    */
   port?: number;
+  /**
+   * Should this ES cluster use SSL?
+   */
   ssl?: boolean;
+  /**
+   * Explicit transport port for a single node to run on, or a string port range to use eg. '9300-9400'
+   * defaults to the transport port from `packages/kbn-test/src/es/es_test_config.ts`
+   */
+  transportPort?: number | string;
 }
 
 export function createTestEsCluster<
@@ -155,13 +163,14 @@ export function createTestEsCluster<
     esJavaOpts,
     clusterName: customClusterName = 'es-test-cluster',
     ssl,
+    transportPort,
   } = options;
 
   const clusterName = `${CI_PARALLEL_PROCESS_PREFIX}${customClusterName}`;
 
   const defaultEsArgs = [
     `cluster.name=${clusterName}`,
-    `transport.port=${esTestConfig.getTransportPort()}`,
+    `transport.port=${transportPort ?? esTestConfig.getTransportPort()}`,
     // For multi-node clusters, we make all nodes master-eligible by default.
     ...(nodes.length > 1
       ? ['discovery.type=zen', `cluster.initial_master_nodes=${nodes.map((n) => n.name).join(',')}`]

diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts
@@ -192,12 +192,17 @@ export const schema = Joi.object()
 
     esTestCluster: Joi.object()
       .keys({
-        license: Joi.string().default('basic'),
+        license: Joi.valid('basic', 'trial', 'gold').default('basic'),
         from: Joi.string().default('snapshot'),
-        serverArgs: Joi.array(),
+        serverArgs: Joi.array().items(Joi.string()),
         esJavaOpts: Joi.string(),
         dataArchive: Joi.string(),
         ssl: Joi.boolean().default(false),
+        ccs: Joi.object().keys({
+          remoteClusterUrl: Joi.string().uri({
+            scheme: /https?/,
+          }),
+        }),
       })
       .default(),
 
@@ -290,6 +295,7 @@ export const schema = Joi.object()
     security: Joi.object()
       .keys({
         roles: Joi.object().default(),
+        remoteEsRoles: Joi.object(),
         defaultRoles: Joi.array()
           .items(Joi.string())
           .when('$primary', {