Merge branch 'main' into siem-explore-148973

package.json

@@ -105,7 +105,7 @@
     "@elastic/apm-rum-react": "^1.4.2",
     "@elastic/charts": "51.3.0",
     "@elastic/datemath": "5.0.3",
-    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.5.0-canary.1",
+    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@8.6.0-canary.3",
     "@elastic/ems-client": "8.3.3",
     "@elastic/eui": "73.0.0",
     "@elastic/filesaver": "1.1.2",

packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/point_in_time_finder.ts

@@ -75,7 +75,7 @@ export class PointInTimeFinder<T = unknown, A = unknown>
     await this.open();
 
     let lastResultsCount: number;
-    let lastHitSortValue: estypes.Id[] | undefined;
+    let lastHitSortValue: estypes.SortResults | undefined;
     do {
       const results = await this.findNext({
         findOptions: this.#findOptions,
@@ -142,7 +142,7 @@ export class PointInTimeFinder<T = unknown, A = unknown>
   }: {
     findOptions: SavedObjectsFindOptions;
     id?: string;
-    searchAfter?: estypes.Id[];
+    searchAfter?: estypes.SortResults;
   }) {
     try {
       return await this.#client.find<T, A>(
@@ -167,7 +167,7 @@ export class PointInTimeFinder<T = unknown, A = unknown>
     }
   }
 
-  private getLastHitSortValue(res: SavedObjectsFindResponse): estypes.Id[] | undefined {
+  private getLastHitSortValue(res: SavedObjectsFindResponse): estypes.SortResults | undefined {
     if (res.saved_objects.length < 1) {
       return undefined;
     }

packages/core/saved-objects/core-saved-objects-api-server-internal/src/lib/search_dsl/search_dsl.ts

@@ -25,7 +25,7 @@ interface GetSearchDslOptions {
   defaultSearchOperator?: SearchOperator;
   searchFields?: string[];
   rootSearchFields?: string[];
-  searchAfter?: estypes.Id[];
+  searchAfter?: estypes.SortResults;
   sortField?: string;
   sortOrder?: estypes.SortOrder;
   namespaces?: string[];

packages/core/saved-objects/core-saved-objects-api-server/src/apis/find.ts

@@ -9,7 +9,7 @@
 import type {
   SortOrder,
   AggregationsAggregationContainer,
-  Id as EsId,
+  SortResults,
 } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import type { SavedObject } from '@kbn/core-saved-objects-common';
 
@@ -68,7 +68,7 @@ export interface SavedObjectsFindOptions {
   /**
    * Use the sort values from the previous page to retrieve the next page of results.
    */
-  searchAfter?: EsId[];
+  searchAfter?: SortResults;
   /**
    * The fields to perform the parsed query against. Unlike the `searchFields` argument, these are expected to be root fields and will not
    * be modified. If used in conjunction with `searchFields`, both are concatenated together.
@@ -176,7 +176,7 @@ export interface SavedObjectsFindResult<T = unknown> extends SavedObject<T> {
    * await savedObjectsClient.closePointInTime(page2.pit_id);
    * ```
    */
-  sort?: string[];
+  sort?: SortResults;
 }
 
 /**

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/core/unused_types.ts

@@ -39,6 +39,8 @@ export const REMOVED_TYPES: string[] = [
   'ui-counter',
   // Deprecated, no longer used since 7.13 https://github.com/elastic/kibana/pull/94923/files
   'application_usage_transactional',
+  // Deprecated, no longer used since 8.7 https://github.com/elastic/kibana/pull/148530
+  'csp_rule',
 ].sort();
 
 // When migrating from the outdated index we use a read query which excludes

packages/core/saved-objects/core-saved-objects-migration-server-internal/src/initial_state.test.ts

@@ -83,6 +83,11 @@ describe('createInitialState', () => {
                   "type": "cases-sub-case",
                 },
               },
+              Object {
+                "term": Object {
+                  "type": "csp_rule",
+                },
+              },
               Object {
                 "term": Object {
                   "type": "file-upload-telemetry",

src/core/server/integration_tests/saved_objects/migrations/check_registered_types.test.ts

@@ -80,7 +80,6 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "connector_token": "fa5301aa5a2914795d3b1b82d0a49939444009da",
         "core-usage-stats": "f40a213da2c597b0de94e364a4326a5a1baa4ca9",
         "csp-rule-template": "d6104585d0b032355c64a7dbf2a834163351cb1c",
-        "csp_rule": "d2bb53ea5d2bdfba1a835ad8956dfcd2b2c32e19",
         "dashboard": "7e37790f802b39c852f905c010e13674e893105a",
         "endpoint:user-artifact": "f94c250a52b30d0a2d32635f8b4c5bdabd1e25c0",
         "endpoint:user-artifact-manifest": "8c14d49a385d5d1307d956aa743ec78de0b2be88",

x-pack/packages/ml/query_utils/README.md

@@ -1,3 +1,3 @@
 # @kbn/ml-query-utils
 
-Query utilities.
+Query utilities to be used in UIs maintained by the @elastic/ml-ui team.

x-pack/packages/ml/query_utils/index.ts

@@ -6,3 +6,5 @@
  */
 
 export { addExcludeFrozenToQuery } from './src/add_exclude_frozen_to_query';
+export { buildBaseFilterCriteria } from './src/build_base_filter_criteria';
+export { getSafeAggregationName } from './src/get_safe_aggregation_name';

x-pack/packages/ml/query_utils/src/add_exclude_frozen_to_query.test.ts

@@ -7,7 +7,7 @@
 
 import { addExcludeFrozenToQuery } from './add_exclude_frozen_to_query';
 
-describe('Util: addExcludeFrozenToQuery()', () => {
+describe('addExcludeFrozenToQuery', () => {
   test('Validation checks.', () => {
     expect(
       addExcludeFrozenToQuery({

x-pack/packages/ml/query_utils/src/build_base_filter_criteria.test.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { buildBaseFilterCriteria } from './build_base_filter_criteria';
+
+describe('buildBaseFilterCriteria', () => {
+  const earliestMs = 1483228800000; // 1 Jan 2017 00:00:00
+  const latestMs = 1485907199000; // 31 Jan 2017 23:59:59
+  const query = {
+    query_string: {
+      query: 'region:sa-east-1',
+      analyze_wildcard: true,
+      default_field: '*',
+    },
+  };
+
+  test('returns correct criteria for time range', () => {
+    expect(buildBaseFilterCriteria('timestamp', earliestMs, latestMs)).toEqual([
+      {
+        range: {
+          timestamp: {
+            gte: earliestMs,
+            lte: latestMs,
+            format: 'epoch_millis',
+          },
+        },
+      },
+    ]);
+  });
+
+  test('returns correct criteria for time range and query', () => {
+    expect(buildBaseFilterCriteria('timestamp', earliestMs, latestMs, query)).toEqual([
+      {
+        range: {
+          timestamp: {
+            gte: earliestMs,
+            lte: latestMs,
+            format: 'epoch_millis',
+          },
+        },
+      },
+      query,
+    ]);
+  });
+});

x-pack/plugins/data_visualizer/common/utils/query_utils.ts → x-pack/packages/ml/query_utils/src/build_base_filter_criteria.ts

@@ -8,19 +8,24 @@
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { Query } from '@kbn/es-query';
 
-/*
- * Contains utility functions for building and processing queries.
+/**
+ * Builds the base filter criteria used in queries,
+ * adding criteria for the time range and an optional query.
+ *
+ * @param timeFieldName - optional time field name of the data view
+ * @param earliestMs - optional earliest timestamp of the selected time range
+ * @param latestMs - optional latest timestamp of the selected time range
+ * @param query - optional query
+ * @returns filter criteria
  */
-
-// Builds the base filter criteria used in queries,
-// adding criteria for the time range and an optional query.
 export function buildBaseFilterCriteria(
   timeFieldName?: string,
   earliestMs?: number,
   latestMs?: number,
   query?: Query['query']
 ): estypes.QueryDslQueryContainer[] {
   const filterCriteria = [];
+
   if (timeFieldName && earliestMs && latestMs) {
     filterCriteria.push({
       range: {
@@ -39,11 +44,3 @@ export function buildBaseFilterCriteria(
 
   return filterCriteria;
 }
-
-// Returns a name which is safe to use in elasticsearch aggregations for the supplied
-// field name. Aggregation names must be alpha-numeric and can only contain '_' and '-' characters,
-// so if the supplied field names contains disallowed characters, the provided index
-// identifier is used to return a safe 'dummy' name in the format 'field_index' e.g. field_0, field_1
-export function getSafeAggregationName(fieldName: string, index: number): string {
-  return fieldName.match(/^[a-zA-Z0-9-_.]+$/) ? fieldName : `field_${index}`;
-}

x-pack/packages/ml/query_utils/src/get_safe_aggregation_name.test.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { getSafeAggregationName } from './get_safe_aggregation_name';
+
+describe('getSafeAggregationName', () => {
+  test('"foo" should be "foo"', () => {
+    expect(getSafeAggregationName('foo', 0)).toBe('foo');
+  });
+  test('"foo.bar" should be "foo.bar"', () => {
+    expect(getSafeAggregationName('foo.bar', 0)).toBe('foo.bar');
+  });
+  test('"foo&bar" should be "field_0"', () => {
+    expect(getSafeAggregationName('foo&bar', 0)).toBe('field_0');
+  });
+});

x-pack/packages/ml/query_utils/src/get_safe_aggregation_name.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Returns a name which is safe to use in elasticsearch aggregations for the supplied
+ * field name. Aggregation names must be alpha-numeric and can only contain '_' and '-' characters,
+ * so if the supplied field names contains disallowed characters, the provided index
+ * identifier is used to return a safe 'dummy' name in the format 'field_index' e.g. field_0, field_1
+ *
+ * @param fieldName - the field name to check
+ * @param index - the index number to be used for the safe aggregation name
+ * @returns safe aggregation name
+ */
+export function getSafeAggregationName(fieldName: string, index: number): string {
+  return fieldName.match(/^[a-zA-Z0-9-_.]+$/) ? fieldName : `field_${index}`;
+}

x-pack/packages/ml/query_utils/tsconfig.json

@@ -17,5 +17,6 @@
   ],
   "kbn_references": [
     "@kbn/ml-is-populated-object",
+    "@kbn/es-query",
   ]
 }

x-pack/plugins/aiops/public/application/utils/query_utils.test.ts → x-pack/plugins/aiops/public/application/utils/build_extended_base_filter_criteria.test.ts

@@ -9,7 +9,7 @@ import type { ChangePoint } from '@kbn/ml-agg-utils';
 
 import type { GroupTableItem } from '../../components/spike_analysis_table/types';
 
-import { buildBaseFilterCriteria } from './query_utils';
+import { buildExtendedBaseFilterCriteria } from './build_extended_base_filter_criteria';
 
 const selectedChangePointMock: ChangePoint = {
   doc_count: 53408,
@@ -41,9 +41,9 @@ const selectedGroupMock: GroupTableItem = {
 };
 
 describe('query_utils', () => {
-  describe('buildBaseFilterCriteria', () => {
+  describe('buildExtendedBaseFilterCriteria', () => {
     it('returns range filter based on minimum supplied arguments', () => {
-      const baseFilterCriteria = buildBaseFilterCriteria('the-time-field-name', 1234, 5678);
+      const baseFilterCriteria = buildExtendedBaseFilterCriteria('the-time-field-name', 1234, 5678);
 
       expect(baseFilterCriteria).toEqual([
         {
@@ -59,7 +59,7 @@ describe('query_utils', () => {
     });
 
     it('returns filters including default query with supplied arguments provided via UI', () => {
-      const baseFilterCriteria = buildBaseFilterCriteria(
+      const baseFilterCriteria = buildExtendedBaseFilterCriteria(
         '@timestamp',
         1640082000012,
         1640103600906,
@@ -81,7 +81,7 @@ describe('query_utils', () => {
     });
 
     it('includes a term filter when including a selectedChangePoint', () => {
-      const baseFilterCriteria = buildBaseFilterCriteria(
+      const baseFilterCriteria = buildExtendedBaseFilterCriteria(
         '@timestamp',
         1640082000012,
         1640103600906,
@@ -105,7 +105,7 @@ describe('query_utils', () => {
     });
 
     it('includes a term filter with must_not when excluding a selectedChangePoint', () => {
-      const baseFilterCriteria = buildBaseFilterCriteria(
+      const baseFilterCriteria = buildExtendedBaseFilterCriteria(
         '@timestamp',
         1640082000012,
         1640103600906,
@@ -130,7 +130,7 @@ describe('query_utils', () => {
     });
 
     it('includes multiple term filters when including a selectedGroupMock', () => {
-      const baseFilterCriteria = buildBaseFilterCriteria(
+      const baseFilterCriteria = buildExtendedBaseFilterCriteria(
         '@timestamp',
         1640082000012,
         1640103600906,
@@ -185,7 +185,7 @@ describe('query_utils', () => {
     });
 
     it('includes a must_not with nested term filters when excluding a selectedGroup', () => {
-      const baseFilterCriteria = buildBaseFilterCriteria(
+      const baseFilterCriteria = buildExtendedBaseFilterCriteria(
         '@timestamp',
         1640082000012,
         1640103600906,