Merge branch 'main' into fleet-metrics

src/plugins/saved_objects_management/server/lib/get_saved_objects_counts.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+import type { SavedObjectsFindOptions } from '@kbn/core-saved-objects-api-server';
+import type { SavedObjectsClientContract } from '@kbn/core/server';
+
+export const getSavedObjectCounts = async ({
+  types,
+  options,
+  client,
+}: {
+  types: string[];
+  options: SavedObjectsFindOptions;
+  client: SavedObjectsClientContract;
+}): Promise<Record<string, number>> => {
+  const body = await client.find<void, { types: estypes.AggregationsStringTermsAggregate }>({
+    ...options,
+    type: types,
+    perPage: 0,
+    aggs: {
+      types: {
+        terms: {
+          field: 'type',
+          size: types.length,
+        },
+      },
+    },
+  });
+
+  const buckets =
+    (body.aggregations?.types?.buckets as estypes.AggregationsStringTermsBucketKeys[]) || [];
+
+  const counts = buckets.reduce((memo, bucket) => {
+    memo[bucket.key] = bucket.doc_count;
+    return memo;
+  }, {} as Record<string, number>);
+
+  return counts;
+};

src/plugins/saved_objects_management/server/lib/index.ts

@@ -9,3 +9,4 @@
 export { toSavedObjectWithMeta } from './to_saved_object_with_meta';
 export { injectMetaAttributes } from './inject_meta_attributes';
 export { findRelationships } from './find_relationships';
+export { getSavedObjectCounts } from './get_saved_objects_counts';

src/plugins/saved_objects_management/server/routes/scroll_count.ts

@@ -10,6 +10,7 @@ import { schema } from '@kbn/config-schema';
 import type { IRouter, SavedObjectsCreatePointInTimeFinderOptions } from '@kbn/core/server';
 import { chain } from 'lodash';
 import type { v1 } from '../../common';
+import { getSavedObjectCounts } from '../lib';
 
 export const registerScrollForCountRoute = (router: IRouter) => {
   router.post(
@@ -44,27 +45,19 @@ export const registerScrollForCountRoute = (router: IRouter) => {
       const client = getClient({ includedHiddenTypes });
       const findOptions: SavedObjectsCreatePointInTimeFinderOptions = {
         type: typesToInclude,
-        perPage: 500,
+        ...(searchString ? { search: `${searchString}*`, searchFields: ['title'] } : {}),
+        ...(references ? { hasReference: references, hasReferenceOperator: 'OR' } : {}),
       };
-      if (searchString) {
-        findOptions.search = `${searchString}*`;
-        findOptions.searchFields = ['title'];
-      }
-      if (references) {
-        findOptions.hasReference = references;
-        findOptions.hasReferenceOperator = 'OR';
-      }
+
+      const rawCounts = await getSavedObjectCounts({
+        types: typesToInclude,
+        client,
+        options: findOptions,
+      });
 
       const counts: Record<string, number> = {};
       for (const type of typesToInclude) {
-        counts[type] = 0;
-      }
-
-      const finder = client.createPointInTimeFinder(findOptions);
-      for await (const { saved_objects: savedObjects } of finder.find()) {
-        for (const { type } of savedObjects) {
-          counts[type]++;
-        }
+        counts[type] = rawCounts[type] ?? 0;
       }
 
       const body: v1.ScrollCountResponseHTTP = counts;

src/plugins/saved_objects_management/tsconfig.json

@@ -28,6 +28,7 @@
     "@kbn/shared-ux-router",
     "@kbn/core-ui-settings-browser-mocks",
     "@kbn/core-ui-settings-browser",
+    "@kbn/core-saved-objects-api-server",
   ],
   "exclude": [
     "target/**/*",

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts

@@ -1124,6 +1124,105 @@ describe('EPM template', () => {
     expect(mappings).toEqual(runtimeFieldMapping);
   });
 
+  it('tests processing scaled_float fields in a dynamic template', () => {
+    const textWithRuntimeFieldsLiteralYml = `
+- name: numeric_labels
+  type: object
+  object_type: scaled_float
+`;
+    const runtimeFieldMapping = {
+      properties: {},
+      dynamic_templates: [
+        {
+          numeric_labels: {
+            match_mapping_type: '*',
+            path_match: 'numeric_labels.*',
+            mapping: {
+              type: 'scaled_float',
+              scaling_factor: 1000,
+            },
+          },
+        },
+      ],
+    };
+    const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
+    const processedFields = processFields(fields);
+    const mappings = generateMappings(processedFields);
+    expect(mappings).toEqual(runtimeFieldMapping);
+  });
+
+  it('tests processing aggregate_metric_double fields in a dynamic template', () => {
+    const textWithRuntimeFieldsLiteralYml = `
+- name: aggregate.*
+  type: aggregate_metric_double
+  metrics: ["min", "max", "sum", "value_count"]
+  default_metric: "max"
+`;
+    const runtimeFieldMapping = {
+      properties: {},
+      dynamic_templates: [
+        {
+          'aggregate.*': {
+            match_mapping_type: '*',
+            path_match: 'aggregate.*',
+            mapping: {
+              type: 'aggregate_metric_double',
+              metrics: ['min', 'max', 'sum', 'value_count'],
+              default_metric: 'max',
+            },
+          },
+        },
+      ],
+    };
+    const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
+    const processedFields = processFields(fields);
+    const mappings = generateMappings(processedFields);
+    expect(mappings).toEqual(runtimeFieldMapping);
+  });
+
+  it('tests processing groub sub fields in a dynamic template', () => {
+    const textWithRuntimeFieldsLiteralYml = `
+- name: group.*.network
+  type: group
+  fields:
+  - name: bytes
+    type: integer
+    metric_type: counter
+`;
+    const runtimeFieldMapping = {
+      properties: {},
+      dynamic_templates: [
+        {
+          'group.*.network.bytes': {
+            match_mapping_type: 'long',
+            path_match: 'group.*.network.bytes',
+            mapping: {
+              type: 'long',
+              time_series_metric: 'counter',
+            },
+          },
+        },
+      ],
+    };
+    const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
+    const processedFields = processFields(fields);
+    const mappings = generateMappings(processedFields);
+    expect(mappings).toEqual(runtimeFieldMapping);
+  });
+
+  it('tests unexpected type for field as dynamic template fails', () => {
+    const textWithRuntimeFieldsLiteralYml = `
+- name: labels.*
+  type: object
+  object_type: constant_keyword
+`;
+    const fields: Field[] = safeLoad(textWithRuntimeFieldsLiteralYml);
+    expect(() => {
+      const processedFields = processFields(fields);
+      generateMappings(processedFields);
+    }).toThrow();
+  });
+
   it('tests priority and index pattern for data stream without dataset_is_prefix', () => {
     const dataStreamDatasetIsPrefixUnset = {
       type: 'metrics',

x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts

@@ -206,7 +206,7 @@ function _generateMappings(
         if (type === 'object' && field.object_type) {
           const pathMatch = path.includes('*') ? path : `${path}.*`;
 
-          let dynProperties: Properties = getDefaultProperties(field);
+          const dynProperties: Properties = getDefaultProperties(field);
           let matchingType: string | undefined;
           switch (field.object_type) {
             case 'keyword':
@@ -216,10 +216,8 @@ function _generateMappings(
             case 'double':
             case 'long':
             case 'boolean':
-              dynProperties = {
-                type: field.object_type,
-                time_series_metric: field.metric_type,
-              };
+              dynProperties.type = field.object_type;
+              dynProperties.time_series_metric = field.metric_type;
               matchingType = field.object_type_mapping_type ?? field.object_type;
             default:
               break;
@@ -258,27 +256,74 @@ function _generateMappings(
             dynProperties = histogram(field);
             matchingType = field.object_type_mapping_type ?? '*';
             break;
+          case 'ip':
+          case 'keyword':
+          case 'match_only_text':
           case 'text':
+          case 'wildcard':
             dynProperties.type = field.object_type;
             matchingType = field.object_type_mapping_type ?? 'string';
             break;
-          case 'keyword':
+          case 'scaled_float':
+            dynProperties = scaledFloat(field);
+            matchingType = field.object_type_mapping_type ?? '*';
+            break;
+          case 'aggregate_metric_double':
             dynProperties.type = field.object_type;
-            matchingType = field.object_type_mapping_type ?? 'string';
+            dynProperties.metrics = field.metrics;
+            dynProperties.default_metric = field.default_metric;
+            matchingType = field.object_type_mapping_type ?? '*';
             break;
-          case 'byte':
           case 'double':
           case 'float':
+          case 'half_float':
+            dynProperties.type = field.object_type;
+            dynProperties.time_series_metric = field.metric_type;
+            matchingType = field.object_type_mapping_type ?? 'double';
+            break;
+          case 'byte':
           case 'long':
           case 'short':
+          case 'unsigned_long':
+            dynProperties.type = field.object_type;
+            dynProperties.time_series_metric = field.metric_type;
+            matchingType = field.object_type_mapping_type ?? 'long';
+            break;
+          case 'integer':
+            // Map integers as long, as in other cases.
+            dynProperties.type = 'long';
+            dynProperties.time_series_metric = field.metric_type;
+            matchingType = field.object_type_mapping_type ?? 'long';
+            break;
           case 'boolean':
-            dynProperties = {
-              type: field.object_type,
-              time_series_metric: field.metric_type,
-            };
+            dynProperties.type = field.object_type;
+            dynProperties.time_series_metric = field.metric_type;
             matchingType = field.object_type_mapping_type ?? field.object_type;
-          default:
             break;
+          case 'group':
+            if (!field?.fields) {
+              break;
+            }
+            const subFields = field.fields.map((subField) => ({
+              ...subField,
+              type: 'object',
+              object_type: subField.object_type ?? subField.type,
+            }));
+            _generateMappings(subFields, {
+              ...ctx,
+              groupFieldName: ctx.groupFieldName
+                ? `${ctx.groupFieldName}.${field.name}`
+                : field.name,
+            });
+            break;
+          case 'flattened':
+            dynProperties.type = field.object_type;
+            matchingType = field.object_type_mapping_type ?? 'object';
+            break;
+          default:
+            throw new Error(
+              `no dynamic mapping generated for field ${path} of type ${field.object_type}`
+            );
         }
 
         if (dynProperties && matchingType) {

x-pack/plugins/log_explorer/common/constants.ts

@@ -10,6 +10,7 @@ export const LOG_EXPLORER_PROFILE_ID = 'log-explorer';
 // Fields constants
 export const TIMESTAMP_FIELD = '@timestamp';
 export const HOST_NAME_FIELD = 'host.name';
+export const LOG_LEVEL_FIELD = 'log.level';
 export const MESSAGE_FIELD = 'message';
 export const SERVICE_NAME_FIELD = 'service.name';
 

x-pack/plugins/log_explorer/public/state_machines/log_explorer_profile/src/url_state_storage_service.ts

@@ -19,6 +19,7 @@ import {
 import {
   DATA_GRID_COLUMNS_PREFERENCES,
   DATA_GRID_DEFAULT_COLUMNS,
+  LOG_LEVEL_FIELD,
 } from '../../../../common/constants';
 import {
   ControlPanelRT,
@@ -182,7 +183,7 @@ export const updateStateContainer =
     LogExplorerProfileEvent
   > =>
   async () => {
-    const { columns, grid, rowHeight } = stateContainer.appState.getState();
+    const { breakdownField, columns, grid, rowHeight } = stateContainer.appState.getState();
     const stateUpdates: DiscoverAppState = {};
 
     // Update data grid columns list
@@ -201,6 +202,9 @@ export const updateStateContainer =
     // Configure rowHeight preference
     stateUpdates.rowHeight = rowHeight ?? ROWS_HEIGHT_OPTIONS.single;
 
+    // Configure breakdown field preference
+    stateUpdates.breakdownField = breakdownField ?? LOG_LEVEL_FIELD;
+
     // Finally batch update state app state
     stateContainer.appState.update(stateUpdates, true);
   };

x-pack/plugins/profiling/server/index.ts

@@ -12,11 +12,15 @@ import { ProfilingPlugin } from './plugin';
 /**
  * These properties are used to create both the Collector and the Symbolizer integrations
  * when Universal Profiling is initialized.
- * As of now Universal Profiling is only availble on Elastic Cloud, so
- * Elastic Cloud will be responsable of filling these properties up and pass it to Kibana.
+ * As of now Universal Profiling is only available on Elastic Cloud, so
+ * Elastic Cloud will fill these properties up and pass it to Kibana.
+ * Note that the list of config options does not encompass all the avaiable entries
+ * offered by the integrations pacakges, but are limited to the ones that
+ * Cloud will make use of.
  */
 const packageInputSchema = schema.object({
   host: schema.maybe(schema.string()),
+  telemetry: schema.maybe(schema.boolean()),
   tls_enabled: schema.maybe(schema.boolean()),
   tls_supported_protocols: schema.maybe(schema.arrayOf(schema.string())),
   tls_certificate_path: schema.maybe(schema.string()),

x-pack/plugins/profiling/server/lib/setup/fleet_policies.test.ts

@@ -72,4 +72,29 @@ describe('getVarsFor', () => {
       tls_key_path: { type: 'text', value: '456' },
     });
   });
+
+  it('returns vars with the telemetry key', () => {
+    const config: PackageInputType = {
+      host: 'example.com',
+      telemetry: true,
+      tls_enabled: true,
+      tls_supported_protocols: ['foo', 'bar'],
+      tls_certificate_path: '123',
+      tls_key_path: '456',
+    };
+
+    const { secret_token: secretToken, ...result } = getVarsFor({
+      config,
+      includeSecretToken: false,
+    });
+    expect(secretToken).toBeUndefined();
+    expect(result).toEqual({
+      host: { type: 'text', value: 'example.com' },
+      telemetry: { type: 'bool', value: true },
+      tls_enabled: { type: 'bool', value: true },
+      tls_supported_protocols: { type: 'text', value: ['foo', 'bar'] },
+      tls_certificate_path: { type: 'text', value: '123' },
+      tls_key_path: { type: 'text', value: '456' },
+    });
+  });
 });