diff --git a/.eslintrc.js b/.eslintrc.js index 96016911e40aa3..f3a4fcf6ecc0d1 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1121,6 +1121,7 @@ module.exports = { 'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}', 'x-pack/plugins/timelines/**/*.{js,mjs,ts,tsx}', 'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}', + 'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}', ], plugins: ['eslint-plugin-node', 'react'], env: { @@ -1218,6 +1219,8 @@ module.exports = { 'x-pack/plugins/security_solution_ess/**/*.{js,mjs,ts,tsx}', 'x-pack/plugins/security_solution_serverless/**/*.{js,mjs,ts,tsx}', 'x-pack/plugins/cases/**/*.{js,mjs,ts,tsx}', + 'x-pack/plugins/ecs_data_quality_dashboard/**/*.{js,mjs,ts,tsx}', + 'packages/kbn-data-stream-adapter/**/*.{js,mjs,ts,tsx}', ], rules: { '@typescript-eslint/consistent-type-imports': 'error', diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0c5559b4d8fd47..f272edf41510f0 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -319,6 +319,7 @@ x-pack/packages/kbn-data-forge @elastic/obs-ux-management-team src/plugins/data @elastic/kibana-visualizations @elastic/kibana-data-discovery test/plugin_functional/plugins/data_search @elastic/kibana-data-discovery packages/kbn-data-service @elastic/kibana-visualizations @elastic/kibana-data-discovery +packages/kbn-data-stream-adapter @elastic/security-threat-hunting-explore src/plugins/data_view_editor @elastic/kibana-data-discovery examples/data_view_field_editor_example @elastic/kibana-data-discovery src/plugins/data_view_field_editor @elastic/kibana-data-discovery diff --git a/package.json b/package.json index cb60182479a905..bc68dac0ffe074 100644 --- a/package.json +++ b/package.json @@ -369,6 +369,7 @@ "@kbn/data-plugin": "link:src/plugins/data", "@kbn/data-search-plugin": "link:test/plugin_functional/plugins/data_search", "@kbn/data-service": "link:packages/kbn-data-service", + "@kbn/data-stream-adapter": "link:packages/kbn-data-stream-adapter", "@kbn/data-view-editor-plugin": "link:src/plugins/data_view_editor", "@kbn/data-view-field-editor-example-plugin": "link:examples/data_view_field_editor_example", "@kbn/data-view-field-editor-plugin": "link:src/plugins/data_view_field_editor", diff --git a/packages/kbn-data-stream-adapter/README.md b/packages/kbn-data-stream-adapter/README.md new file mode 100644 index 00000000000000..04a3d854aced7b --- /dev/null +++ b/packages/kbn-data-stream-adapter/README.md @@ -0,0 +1,69 @@ +# @kbn/data-stream-adapter + +Utility library for Elasticsearch data stream management. + +## DataStreamAdapter + +Manage single data streams. Example: + +``` +// Setup +const dataStream = new DataStreamAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); + +dataStream.setComponentTemplate({ + name: 'awesome-component-template', + fieldMap: { + 'awesome.field1: { type: 'keyword', required: true }, + 'awesome.nested.field2: { type: 'number', required: false }, + // ... + }, +}); + +dataStream.setIndexTemplate({ + name: 'awesome-index-template', + componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], + template: { + lifecycle: { + data_retention: '5d', + }, + }, +}); + +// Start +await dataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and the data stream, or updates existing. +``` + + +## DataStreamSpacesAdapter + +Manage data streams per space. Example: + +``` +// Setup +const spacesDataStream = new DataStreamSpacesAdapter('my-awesome-datastream', { kibanaVersion: '8.12.1' }); + +spacesDataStream.setComponentTemplate({ + name: 'awesome-component-template', + fieldMap: { + 'awesome.field1: { type: 'keyword', required: true }, + 'awesome.nested.field2: { type: 'number', required: false }, + // ... + }, +}); + +spacesDataStream.setIndexTemplate({ + name: 'awesome-index-template', + componentTemplateRefs: ['awesome-component-template', 'ecs-component-template'], + template: { + lifecycle: { + data_retention: '5d', + }, + }, +}); + +// Start +await spacesDataStream.install({ logger, esClient, pluginStop$ }); // Installs templates and updates existing data streams. + +// Create a space data stream on the fly +await spacesDataStream.installSpace('space2'); // creates 'my-awesome-datastream-space2' data stream if it does not exist. +``` diff --git a/packages/kbn-data-stream-adapter/index.ts b/packages/kbn-data-stream-adapter/index.ts new file mode 100644 index 00000000000000..808145be4f12eb --- /dev/null +++ b/packages/kbn-data-stream-adapter/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { DataStreamAdapter } from './src/data_stream_adapter'; +export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter'; +export { retryTransientEsErrors } from './src/retry_transient_es_errors'; +export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map'; + +export type { + DataStreamAdapterParams, + SetComponentTemplateParams, + SetIndexTemplateParams, + InstallParams, +} from './src/data_stream_adapter'; +export * from './src/field_maps/types'; diff --git a/packages/kbn-data-stream-adapter/jest.config.js b/packages/kbn-data-stream-adapter/jest.config.js new file mode 100644 index 00000000000000..48b717249e3533 --- /dev/null +++ b/packages/kbn-data-stream-adapter/jest.config.js @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../..', + roots: ['/packages/kbn-data-stream-adapter'], +}; diff --git a/packages/kbn-data-stream-adapter/kibana.jsonc b/packages/kbn-data-stream-adapter/kibana.jsonc new file mode 100644 index 00000000000000..99cbb458a8517c --- /dev/null +++ b/packages/kbn-data-stream-adapter/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/data-stream-adapter", + "owner": "@elastic/security-threat-hunting-explore" +} diff --git a/packages/kbn-data-stream-adapter/package.json b/packages/kbn-data-stream-adapter/package.json new file mode 100644 index 00000000000000..80b16c25ac217e --- /dev/null +++ b/packages/kbn-data-stream-adapter/package.json @@ -0,0 +1,7 @@ +{ + "name": "@kbn/data-stream-adapter", + "version": "1.0.0", + "description": "Utility library for Elasticsearch Data Stream management", + "license": "SSPL-1.0 OR Elastic License 2.0", + "private": true +} diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts new file mode 100644 index 00000000000000..78fabd29d1ae88 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.test.ts @@ -0,0 +1,290 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import type { DiagnosticResult } from '@elastic/elasticsearch'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; + +const randomDelayMultiplier = 0.01; +const logger = loggingSystemMock.createLogger(); +const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +const componentTemplate = { + name: 'test-mappings', + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: false, + properties: { + foo: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, +}; + +describe('createOrUpdateComponentTemplate', () => { + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); + }); + + it(`should call esClient to put component template`, async () => { + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledWith(componentTemplate); + }); + + it(`should retry on transient ES errors`, async () => { + clusterClient.cluster.putComponentTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + }); + + it(`should log and throw error if max retries exceeded`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValue( + new EsErrors.ConnectionError('foo') + ); + await expect(() => + createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing component template test-mappings - foo` + ); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + }); + + it(`should log and throw error if ES throws error`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValue(new Error('generic error')); + + await expect(() => + createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing component template test-mappings - generic error` + ); + }); + + it(`should update index template field limit and retry if putTemplate throws error with field limit error`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError({ + body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded', + } as DiagnosticResult) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['test-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 1800, + }, + mappings: { + dynamic: false, + }, + }, + }, + }; + + clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [existingIndexTemplate], + }); + + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500'); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ + name: existingIndexTemplate.name, + body: { + ...existingIndexTemplate.index_template, + template: { + ...existingIndexTemplate.index_template.template, + settings: { + ...existingIndexTemplate.index_template.template?.settings, + 'index.mapping.total_fields.limit': 2500, + }, + }, + }, + }); + }); + + it(`should update index template field limit and retry if putTemplate throws error with field limit error when there are malformed index templates`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError({ + body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded', + } as DiagnosticResult) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['test-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 1800, + }, + mappings: { + dynamic: false, + }, + }, + }, + }; + + clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [ + existingIndexTemplate, + { + name: 'lyndon', + // @ts-expect-error + index_template: { + index_patterns: ['intel*'], + }, + }, + { + name: 'sample_ds', + // @ts-expect-error + index_template: { + index_patterns: ['sample_ds-*'], + data_stream: { + hidden: false, + allow_custom_routing: false, + }, + }, + }, + ], + }); + + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500'); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ + name: existingIndexTemplate.name, + body: { + ...existingIndexTemplate.index_template, + template: { + ...existingIndexTemplate.index_template.template, + settings: { + ...existingIndexTemplate.index_template.template?.settings, + 'index.mapping.total_fields.limit': 2500, + }, + }, + }, + }); + }); + + it(`should retry getIndexTemplate and putIndexTemplate on transient ES errors`, async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError({ + body: 'illegal_argument_exception: Limit of total fields [1800] has been exceeded', + } as DiagnosticResult) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['test-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 1800, + }, + mappings: { + dynamic: false, + }, + }, + }, + }; + clusterClient.indices.getIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValueOnce({ + index_templates: [existingIndexTemplate], + }); + clusterClient.indices.putIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + await createOrUpdateComponentTemplate({ + logger, + esClient: clusterClient, + template: componentTemplate, + totalFieldsLimit: 2500, + }); + + expect(logger.info).toHaveBeenCalledWith('Updating total_fields.limit from 1800 to 2500'); + expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(3); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts new file mode 100644 index 00000000000000..c63dedaae6ea14 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_component_template.ts @@ -0,0 +1,118 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + ClusterPutComponentTemplateRequest, + IndicesGetIndexTemplateIndexTemplateItem, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { asyncForEach } from '@kbn/std'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +interface CreateOrUpdateComponentTemplateOpts { + logger: Logger; + esClient: ElasticsearchClient; + template: ClusterPutComponentTemplateRequest; + totalFieldsLimit: number; +} + +const putIndexTemplateTotalFieldsLimitUsingComponentTemplate = async ( + esClient: ElasticsearchClient, + componentTemplateName: string, + totalFieldsLimit: number, + logger: Logger +) => { + // Get all index templates and filter down to just the ones referencing this component template + const { index_templates: indexTemplates } = await retryTransientEsErrors( + () => esClient.indices.getIndexTemplate(), + { logger } + ); + const indexTemplatesUsingComponentTemplate = (indexTemplates ?? []).filter( + (indexTemplate: IndicesGetIndexTemplateIndexTemplateItem) => + (indexTemplate.index_template?.composed_of ?? []).includes(componentTemplateName) + ); + + await asyncForEach( + indexTemplatesUsingComponentTemplate, + async (template: IndicesGetIndexTemplateIndexTemplateItem) => { + await retryTransientEsErrors( + () => + esClient.indices.putIndexTemplate({ + name: template.name, + body: { + ...template.index_template, + template: { + ...template.index_template.template, + settings: { + ...template.index_template.template?.settings, + 'index.mapping.total_fields.limit': totalFieldsLimit, + }, + }, + }, + }), + { logger } + ); + } + ); +}; + +const createOrUpdateComponentTemplateHelper = async ( + esClient: ElasticsearchClient, + template: ClusterPutComponentTemplateRequest, + totalFieldsLimit: number, + logger: Logger +) => { + try { + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { logger }); + } catch (error) { + const limitErrorMatch = error.message.match( + /Limit of total fields \[(\d+)\] has been exceeded/ + ); + if (limitErrorMatch != null) { + // This error message occurs when there is an index template using this component template + // that contains a field limit setting that using this component template exceeds + // Specifically, this can happen for the ECS component template when we add new fields + // to adhere to the ECS spec. Individual index templates specify field limits so if the + // number of new ECS fields pushes the composed mapping above the limit, this error will + // occur. We have to update the field limit inside the index template now otherwise we + // can never update the component template + + logger.info(`Updating total_fields.limit from ${limitErrorMatch[1]} to ${totalFieldsLimit}`); + + await putIndexTemplateTotalFieldsLimitUsingComponentTemplate( + esClient, + template.name, + totalFieldsLimit, + logger + ); + + // Try to update the component template again + await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { + logger, + }); + } else { + throw error; + } + } +}; + +export const createOrUpdateComponentTemplate = async ({ + logger, + esClient, + template, + totalFieldsLimit, +}: CreateOrUpdateComponentTemplateOpts) => { + logger.info(`Installing component template ${template.name}`); + + try { + await createOrUpdateComponentTemplateHelper(esClient, template, totalFieldsLimit, logger); + } catch (err) { + logger.error(`Error installing component template ${template.name} - ${err.message}`); + throw err; + } +}; diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts new file mode 100644 index 00000000000000..cc587dcaebfad3 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types'; +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { + updateDataStreams, + createDataStream, + createOrUpdateDataStream, +} from './create_or_update_data_stream'; + +const logger = loggingSystemMock.createLogger(); +const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +esClient.indices.putMapping.mockResolvedValue({ acknowledged: true }); +esClient.indices.putSettings.mockResolvedValue({ acknowledged: true }); + +const simulateIndexTemplateResponse = { template: { mappings: { is_managed: true } } }; +esClient.indices.simulateIndexTemplate.mockResolvedValue(simulateIndexTemplateResponse); + +const name = 'test_data_stream'; +const totalFieldsLimit = 1000; + +describe('updateDataStreams', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should update data streams`, async () => { + const dataStreamName = 'test_data_stream-default'; + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: dataStreamName } as IndicesDataStream], + }); + + await updateDataStreams({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' }); + + expect(esClient.indices.putSettings).toHaveBeenCalledWith({ + index: dataStreamName, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ + name: dataStreamName, + }); + expect(esClient.indices.putMapping).toHaveBeenCalledWith({ + index: dataStreamName, + body: simulateIndexTemplateResponse.template.mappings, + }); + }); + + it(`should update multiple data streams`, async () => { + const dataStreamName1 = 'test_data_stream-1'; + const dataStreamName2 = 'test_data_stream-2'; + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: dataStreamName1 }, { name: dataStreamName2 }] as IndicesDataStream[], + }); + + await updateDataStreams({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(esClient.indices.putMapping).toHaveBeenCalledTimes(2); + }); + + it(`should not update data streams when not exist`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); + + await updateDataStreams({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.putSettings).not.toHaveBeenCalled(); + expect(esClient.indices.simulateIndexTemplate).not.toHaveBeenCalled(); + expect(esClient.indices.putMapping).not.toHaveBeenCalled(); + }); +}); + +describe('createDataStream', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should create data stream`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); + + await createDataStream({ + esClient, + logger, + name, + }); + + expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name }); + }); + + it(`should not create data stream if already exists`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream], + }); + + await createDataStream({ + esClient, + logger, + name, + }); + + expect(esClient.indices.createDataStream).not.toHaveBeenCalled(); + }); +}); + +describe('createOrUpdateDataStream', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it(`should create data stream if not exists`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] }); + + await createDataStream({ + esClient, + logger, + name, + }); + + expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name }); + }); + + it(`should update data stream if already exists`, async () => { + esClient.indices.getDataStream.mockResolvedValueOnce({ + data_streams: [{ name: 'test_data_stream-default' } as IndicesDataStream], + }); + + await createOrUpdateDataStream({ + esClient, + logger, + name, + totalFieldsLimit, + }); + + expect(esClient.indices.getDataStream).toHaveBeenCalledWith({ name, expand_wildcards: 'all' }); + + expect(esClient.indices.putSettings).toHaveBeenCalledWith({ + index: name, + body: { 'index.mapping.total_fields.limit': totalFieldsLimit }, + }); + expect(esClient.indices.simulateIndexTemplate).toHaveBeenCalledWith({ + name, + }); + expect(esClient.indices.putMapping).toHaveBeenCalledWith({ + index: name, + body: simulateIndexTemplateResponse.template.mappings, + }); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts new file mode 100644 index 00000000000000..5cff6005ea8e0c --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { get } from 'lodash'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +interface UpdateIndexMappingsOpts { + logger: Logger; + esClient: ElasticsearchClient; + indexNames: string[]; + totalFieldsLimit: number; +} + +interface UpdateIndexOpts { + logger: Logger; + esClient: ElasticsearchClient; + indexName: string; + totalFieldsLimit: number; +} + +const updateTotalFieldLimitSetting = async ({ + logger, + esClient, + indexName, + totalFieldsLimit, +}: UpdateIndexOpts) => { + logger.debug(`Updating total field limit setting for ${indexName} data stream.`); + + try { + const body = { 'index.mapping.total_fields.limit': totalFieldsLimit }; + await retryTransientEsErrors(() => esClient.indices.putSettings({ index: indexName, body }), { + logger, + }); + } catch (err) { + logger.error( + `Failed to PUT index.mapping.total_fields.limit settings for ${indexName}: ${err.message}` + ); + throw err; + } +}; + +// This will update the mappings but *not* the settings. This +// is due to the fact settings can be classed as dynamic and static, and static +// updates will fail on an index that isn't closed. New settings *will* be applied as part +// of the ILM policy rollovers. More info: https://github.com/elastic/kibana/pull/113389#issuecomment-940152654 +const updateMapping = async ({ logger, esClient, indexName }: UpdateIndexOpts) => { + logger.debug(`Updating mappings for ${indexName} data stream.`); + + let simulatedIndexMapping: IndicesSimulateIndexTemplateResponse; + try { + simulatedIndexMapping = await retryTransientEsErrors( + () => esClient.indices.simulateIndexTemplate({ name: indexName }), + { logger } + ); + } catch (err) { + logger.error( + `Ignored PUT mappings for ${indexName}; error generating simulated mappings: ${err.message}` + ); + return; + } + + const simulatedMapping = get(simulatedIndexMapping, ['template', 'mappings']); + + if (simulatedMapping == null) { + logger.error(`Ignored PUT mappings for ${indexName}; simulated mappings were empty`); + return; + } + + try { + await retryTransientEsErrors( + () => esClient.indices.putMapping({ index: indexName, body: simulatedMapping }), + { logger } + ); + } catch (err) { + logger.error(`Failed to PUT mapping for ${indexName}: ${err.message}`); + throw err; + } +}; +/** + * Updates the data stream mapping and total field limit setting + */ +const updateDataStreamMappings = async ({ + logger, + esClient, + totalFieldsLimit, + indexNames, +}: UpdateIndexMappingsOpts) => { + // Update total field limit setting of found indices + // Other index setting changes are not updated at this time + await Promise.all( + indexNames.map((indexName) => + updateTotalFieldLimitSetting({ logger, esClient, totalFieldsLimit, indexName }) + ) + ); + // Update mappings of the found indices. + await Promise.all( + indexNames.map((indexName) => updateMapping({ logger, esClient, totalFieldsLimit, indexName })) + ); +}; + +export interface CreateOrUpdateDataStreamParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; +} + +export async function createOrUpdateDataStream({ + logger, + esClient, + name, + totalFieldsLimit, +}: CreateOrUpdateDataStreamParams): Promise { + logger.info(`Creating data stream - ${name}`); + + // check if data stream exists + let dataStreamExists = false; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), + { logger } + ); + dataStreamExists = response.data_streams.length > 0; + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching data stream for ${name} - ${error.message}`); + throw error; + } + } + + // if a data stream exists, update the underlying mapping + if (dataStreamExists) { + await updateDataStreamMappings({ + logger, + esClient, + indexNames: [name], + totalFieldsLimit, + }); + } else { + try { + await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Error creating data stream ${name} - ${error.message}`); + throw error; + } + } + } +} + +export interface CreateDataStreamParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; +} + +export async function createDataStream({ + logger, + esClient, + name, +}: CreateDataStreamParams): Promise { + logger.info(`Creating data stream - ${name}`); + + // check if data stream exists + let dataStreamExists = false; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), + { logger } + ); + dataStreamExists = response.data_streams.length > 0; + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching data stream for ${name} - ${error.message}`); + throw error; + } + } + + // return if data stream already created + if (dataStreamExists) { + return; + } + + try { + await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger }); + } catch (error) { + if (error?.meta?.body?.error?.type !== 'resource_already_exists_exception') { + logger.error(`Error creating data stream ${name} - ${error.message}`); + throw error; + } + } +} + +export interface CreateOrUpdateSpacesDataStreamParams { + name: string; + logger: Logger; + esClient: ElasticsearchClient; + totalFieldsLimit: number; +} + +export async function updateDataStreams({ + logger, + esClient, + name, + totalFieldsLimit, +}: CreateOrUpdateSpacesDataStreamParams): Promise { + logger.info(`Updating data streams - ${name}`); + + // check if data stream exists + let dataStreams: IndicesDataStream[] = []; + try { + const response = await retryTransientEsErrors( + () => esClient.indices.getDataStream({ name, expand_wildcards: 'all' }), + { logger } + ); + dataStreams = response.data_streams; + } catch (error) { + if (error?.statusCode !== 404) { + logger.error(`Error fetching data stream for ${name} - ${error.message}`); + throw error; + } + } + if (dataStreams.length > 0) { + await updateDataStreamMappings({ + logger, + esClient, + totalFieldsLimit, + indexNames: dataStreams.map((dataStream) => dataStream.name), + }); + } +} diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts new file mode 100644 index 00000000000000..cb3b6e77a02b5f --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.test.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { errors as EsErrors } from '@elastic/elasticsearch'; +import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; + +const randomDelayMultiplier = 0.01; +const logger = loggingSystemMock.createLogger(); +const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser; + +const getIndexTemplate = (namespace: string = 'default', useDataStream: boolean = false) => ({ + name: `.alerts-test.alerts-${namespace}-index-template`, + body: { + _meta: { + kibana: { + version: '8.6.1', + }, + managed: true, + namespace, + }, + composed_of: ['mappings1', 'framework-mappings'], + index_patterns: [`.internal.alerts-test.alerts-${namespace}-*`], + template: { + mappings: { + _meta: { + kibana: { + version: '8.6.1', + }, + managed: true, + namespace, + }, + dynamic: false, + }, + settings: { + auto_expand_replicas: '0-1', + hidden: true, + ...(useDataStream + ? {} + : { + 'index.lifecycle': { + name: 'test-ilm-policy', + rollover_alias: `.alerts-test.alerts-${namespace}`, + }, + }), + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': 2500, + }, + }, + priority: namespace.length, + }, +}); + +const simulateTemplateResponse = { + template: { + aliases: { + alias_name_1: { + is_hidden: true, + }, + alias_name_2: { + is_hidden: true, + }, + }, + mappings: { enabled: false }, + settings: {}, + }, +}; + +describe('createOrUpdateIndexTemplate', () => { + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(global.Math, 'random').mockReturnValue(randomDelayMultiplier); + }); + + it(`should call esClient to put index template`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); + + expect(esClient.indices.simulateTemplate).toHaveBeenCalledWith(getIndexTemplate()); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledWith(getIndexTemplate()); + }); + + it(`should retry on transient ES errors`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockResolvedValue({ acknowledged: true }); + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); + + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(3); + }); + + it(`should retry simulateTemplate on transient ES errors`, async () => { + esClient.indices.simulateTemplate + .mockRejectedValueOnce(new EsErrors.ConnectionError('foo')) + .mockRejectedValueOnce(new EsErrors.TimeoutError('timeout')) + .mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate.mockResolvedValue({ acknowledged: true }); + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate }); + + expect(esClient.indices.simulateTemplate).toHaveBeenCalledTimes(3); + }); + + it(`should log and throw error if max retries exceeded`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate.mockRejectedValue(new EsErrors.ConnectionError('foo')); + await expect(() => + createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"foo"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing index template .alerts-test.alerts-default-index-template - foo`, + expect.any(Error) + ); + expect(esClient.indices.putIndexTemplate).toHaveBeenCalledTimes(4); + }); + + it(`should log and throw error if ES throws error`, async () => { + esClient.indices.simulateTemplate.mockImplementation(async () => simulateTemplateResponse); + esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error')); + + await expect(() => + createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) + ).rejects.toThrowErrorMatchingInlineSnapshot(`"generic error"`); + + expect(logger.error).toHaveBeenCalledWith( + `Error installing index template .alerts-test.alerts-default-index-template - generic error`, + expect.any(Error) + ); + }); + + it(`should log and return without updating template if simulate throws error`, async () => { + esClient.indices.simulateTemplate.mockRejectedValue(new Error('simulate error')); + esClient.indices.putIndexTemplate.mockRejectedValue(new Error('generic error')); + + await createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }); + + expect(logger.error).toHaveBeenCalledWith( + `Failed to simulate index template mappings for .alerts-test.alerts-default-index-template; not applying mappings - simulate error`, + expect.any(Error) + ); + expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + }); + + it(`should throw error if simulate returns empty mappings`, async () => { + esClient.indices.simulateTemplate.mockImplementationOnce(async () => ({ + ...simulateTemplateResponse, + template: { + ...simulateTemplateResponse.template, + mappings: {}, + }, + })); + + await expect(() => + createOrUpdateIndexTemplate({ logger, esClient, template: getIndexTemplate() }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"No mappings would be generated for .alerts-test.alerts-default-index-template, possibly due to failed/misconfigured bootstrapping"` + ); + expect(esClient.indices.putIndexTemplate).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts new file mode 100644 index 00000000000000..e9d5d589c55d62 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/create_or_update_index_template.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndicesPutIndexTemplateRequest, + MappingTypeMapping, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import { isEmpty } from 'lodash/fp'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +interface CreateOrUpdateIndexTemplateOpts { + logger: Logger; + esClient: ElasticsearchClient; + template: IndicesPutIndexTemplateRequest; +} + +/** + * Installs index template that uses installed component template + * Prior to installation, simulates the installation to check for possible + * conflicts. Simulate should return an empty mapping if a template + * conflicts with an already installed template. + */ +export const createOrUpdateIndexTemplate = async ({ + logger, + esClient, + template, +}: CreateOrUpdateIndexTemplateOpts) => { + logger.info(`Installing index template ${template.name}`); + + let mappings: MappingTypeMapping = {}; + try { + // Simulate the index template to proactively identify any issues with the mapping + const simulateResponse = await retryTransientEsErrors( + () => esClient.indices.simulateTemplate(template), + { logger } + ); + mappings = simulateResponse.template.mappings; + } catch (err) { + logger.error( + `Failed to simulate index template mappings for ${template.name}; not applying mappings - ${err.message}`, + err + ); + return; + } + + if (isEmpty(mappings)) { + throw new Error( + `No mappings would be generated for ${template.name}, possibly due to failed/misconfigured bootstrapping` + ); + } + + try { + await retryTransientEsErrors(() => esClient.indices.putIndexTemplate(template), { + logger, + }); + } catch (err) { + logger.error(`Error installing index template ${template.name} - ${err.message}`, err); + throw err; + } +}; diff --git a/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts new file mode 100644 index 00000000000000..3b3e2958eb46ae --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/data_stream_adapter.ts @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + ClusterPutComponentTemplateRequest, + IndicesIndexSettings, + IndicesPutIndexTemplateIndexTemplateMapping, + IndicesPutIndexTemplateRequest, +} from '@elastic/elasticsearch/lib/api/types'; +import type { Logger, ElasticsearchClient } from '@kbn/core/server'; +import type { Subject } from 'rxjs'; +import type { FieldMap } from './field_maps/types'; +import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; +import { createOrUpdateDataStream } from './create_or_update_data_stream'; +import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; +import { InstallShutdownError, installWithTimeout } from './install_with_timeout'; +import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils'; + +export interface DataStreamAdapterParams { + kibanaVersion: string; + totalFieldsLimit?: number; +} +export interface SetComponentTemplateParams { + name: string; + fieldMap: FieldMap; + settings?: IndicesIndexSettings; + dynamic?: 'strict' | boolean; +} +export interface SetIndexTemplateParams { + name: string; + componentTemplateRefs?: string[]; + namespace?: string; + template?: IndicesPutIndexTemplateIndexTemplateMapping; + hidden?: boolean; +} + +export interface GetInstallFnParams { + logger: Logger; + pluginStop$: Subject; + tasksTimeoutMs?: number; +} +export interface InstallParams { + logger: Logger; + esClient: ElasticsearchClient | Promise; + pluginStop$: Subject; + tasksTimeoutMs?: number; +} + +const DEFAULT_FIELDS_LIMIT = 2500; + +export class DataStreamAdapter { + protected readonly kibanaVersion: string; + protected readonly totalFieldsLimit: number; + protected componentTemplates: ClusterPutComponentTemplateRequest[] = []; + protected indexTemplates: IndicesPutIndexTemplateRequest[] = []; + protected installed: boolean; + + constructor(protected readonly name: string, options: DataStreamAdapterParams) { + this.installed = false; + this.kibanaVersion = options.kibanaVersion; + this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT; + } + + public setComponentTemplate(params: SetComponentTemplateParams) { + if (this.installed) { + throw new Error('Cannot set component template after install'); + } + this.componentTemplates.push(getComponentTemplate(params)); + } + + public setIndexTemplate(params: SetIndexTemplateParams) { + if (this.installed) { + throw new Error('Cannot set index template after install'); + } + this.indexTemplates.push( + getIndexTemplate({ + ...params, + indexPatterns: [this.name], + kibanaVersion: this.kibanaVersion, + totalFieldsLimit: this.totalFieldsLimit, + }) + ); + } + + protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) { + return async (promise: Promise, description?: string): Promise => { + try { + await installWithTimeout({ + installFn: () => promise, + description, + timeoutMs: tasksTimeoutMs, + pluginStop$, + }); + } catch (err) { + if (err instanceof InstallShutdownError) { + logger.info(err.message); + } else { + throw err; + } + } + }; + } + + public async install({ + logger, + esClient: esClientToResolve, + pluginStop$, + tasksTimeoutMs, + }: InstallParams) { + this.installed = true; + + const esClient = await esClientToResolve; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + + // Install component templates in parallel + await Promise.all( + this.componentTemplates.map((componentTemplate) => + installFn( + createOrUpdateComponentTemplate({ + template: componentTemplate, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `${componentTemplate.name} component template` + ) + ) + ); + + // Install index templates in parallel + await Promise.all( + this.indexTemplates.map((indexTemplate) => + installFn( + createOrUpdateIndexTemplate({ + template: indexTemplate, + esClient, + logger, + }), + `${indexTemplate.name} index template` + ) + ) + ); + + // create data stream when everything is ready + await installFn( + createOrUpdateDataStream({ + name: this.name, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `${this.name} data stream` + ); + } +} diff --git a/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts new file mode 100644 index 00000000000000..5daad080d47203 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { createOrUpdateComponentTemplate } from './create_or_update_component_template'; +import { createDataStream, updateDataStreams } from './create_or_update_data_stream'; +import { createOrUpdateIndexTemplate } from './create_or_update_index_template'; +import { + DataStreamAdapter, + type DataStreamAdapterParams, + type InstallParams, +} from './data_stream_adapter'; + +export class DataStreamSpacesAdapter extends DataStreamAdapter { + private installedSpaceDataStreamName: Map>; + private _installSpace?: (spaceId: string) => Promise; + + constructor(private readonly prefix: string, options: DataStreamAdapterParams) { + super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names + this.installedSpaceDataStreamName = new Map(); + } + + public async install({ + logger, + esClient: esClientToResolve, + pluginStop$, + tasksTimeoutMs, + }: InstallParams) { + this.installed = true; + + const esClient = await esClientToResolve; + const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs }); + + // Install component templates in parallel + await Promise.all( + this.componentTemplates.map((componentTemplate) => + installFn( + createOrUpdateComponentTemplate({ + template: componentTemplate, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `create or update ${componentTemplate.name} component template` + ) + ) + ); + + // Install index templates in parallel + await Promise.all( + this.indexTemplates.map((indexTemplate) => + installFn( + createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }), + `create or update ${indexTemplate.name} index template` + ) + ) + ); + + // Update existing space data streams + await installFn( + updateDataStreams({ + name: `${this.prefix}-*`, + esClient, + logger, + totalFieldsLimit: this.totalFieldsLimit, + }), + `update space data streams` + ); + + // define function to install data stream for spaces on demand + this._installSpace = async (spaceId: string) => { + const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId); + if (existingInstallPromise) { + return existingInstallPromise; + } + const name = `${this.prefix}-${spaceId}`; + const installPromise = installFn( + createDataStream({ name, esClient, logger }), + `create ${name} data stream` + ).then(() => name); + + this.installedSpaceDataStreamName.set(spaceId, installPromise); + return installPromise; + }; + } + + public async installSpace(spaceId: string): Promise { + if (!this._installSpace) { + throw new Error('Cannot installSpace before install'); + } + return this._installSpace(spaceId); + } + + public async getInstalledSpaceName(spaceId: string): Promise { + return this.installedSpaceDataStreamName.get(spaceId); + } +} diff --git a/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts b/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts new file mode 100644 index 00000000000000..17e8af1da78871 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/ecs_field_map.ts @@ -0,0 +1,88 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EcsFlat } from '@kbn/ecs'; +import type { EcsMetadata, FieldMap } from './types'; + +const EXCLUDED_TYPES = ['constant_keyword']; + +// ECS fields that have reached Stage 2 in the RFC process +// are included in the generated Yaml but are still considered +// experimental. Some are correctly marked as beta but most are +// not. + +// More about the RFC stages here: https://elastic.github.io/ecs/stages.html + +// The following RFCS are currently in stage 2: +// https://github.com/elastic/ecs/blob/main/rfcs/text/0027-faas-fields.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0035-tty-output.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0037-host-metrics.md +// https://github.com/elastic/ecs/blob/main/rfcs/text/0040-volume-device.md + +// Fields from these RFCs that are not already in the ECS component template +// as of 8.11 are manually identified as experimental below. +// The next time this list is updated, we should check the above list of RFCs to +// see if any have moved to Stage 3 and remove them from the list and check if +// there are any new stage 2 RFCs with fields we should exclude as experimental. + +const EXPERIMENTAL_FIELDS = [ + 'faas.trigger', // this was previously mapped as nested but changed to object + 'faas.trigger.request_id', + 'faas.trigger.type', + 'host.cpu.system.norm.pct', + 'host.cpu.user.norm.pct', + 'host.fsstats.total_size.total', + 'host.fsstats.total_size.used', + 'host.fsstats.total_size.used.pct', + 'host.load.norm.1', + 'host.load.norm.5', + 'host.load.norm.15', + 'host.memory.actual.used.bytes', + 'host.memory.actual.used.pct', + 'host.memory.total', + 'process.io.bytes', + 'volume.bus_type', + 'volume.default_access', + 'volume.device_name', + 'volume.device_type', + 'volume.dos_name', + 'volume.file_system_type', + 'volume.mount_name', + 'volume.nt_name', + 'volume.product_id', + 'volume.product_name', + 'volume.removable', + 'volume.serial_number', + 'volume.size', + 'volume.vendor_id', + 'volume.vendor_name', + 'volume.writable', +]; + +export const ecsFieldMap: FieldMap = Object.fromEntries( + Object.entries(EcsFlat) + .filter( + ([key, value]) => !EXCLUDED_TYPES.includes(value.type) && !EXPERIMENTAL_FIELDS.includes(key) + ) + .map(([key, _]) => { + const value: EcsMetadata = EcsFlat[key as keyof typeof EcsFlat]; + return [ + key, + { + type: value.type, + array: value.normalize.includes('array'), + required: !!value.required, + ...(value.scaling_factor ? { scaling_factor: value.scaling_factor } : {}), + ...(value.ignore_above ? { ignore_above: value.ignore_above } : {}), + ...(value.multi_fields ? { multi_fields: value.multi_fields } : {}), + }, + ]; + }) +); + +export type EcsFieldMap = typeof ecsFieldMap; diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts new file mode 100644 index 00000000000000..e851bdc21d01b9 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.test.ts @@ -0,0 +1,393 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils'; +import { mappingFromFieldMap } from './mapping_from_field_map'; + +export const testFieldMap: FieldMap = { + date_field: { + type: 'date', + array: false, + required: true, + }, + keyword_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + long_field: { + type: 'long', + array: false, + required: false, + }, + multifield_field: { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + multi_fields: [ + { + flat_name: 'multifield_field.text', + name: 'text', + type: 'match_only_text', + }, + ], + }, + geopoint_field: { + type: 'geo_point', + array: false, + required: false, + }, + ip_field: { + type: 'ip', + array: false, + required: false, + }, + array_field: { + type: 'keyword', + array: true, + required: false, + ignore_above: 1024, + }, + nested_array_field: { + type: 'nested', + array: false, + required: false, + }, + 'nested_array_field.field1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'nested_array_field.field2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + scaled_float_field: { + type: 'scaled_float', + array: false, + required: false, + scaling_factor: 1000, + }, + constant_keyword_field: { + type: 'constant_keyword', + array: false, + required: false, + }, + 'parent_field.child1': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + 'parent_field.child2': { + type: 'keyword', + array: false, + required: false, + ignore_above: 1024, + }, + unmapped_object: { + type: 'object', + required: false, + enabled: false, + }, + formatted_field: { + type: 'date_range', + required: false, + format: 'epoch_millis||strict_date_optional_time', + }, +}; +export const expectedTestMapping = { + properties: { + array_field: { + ignore_above: 1024, + type: 'keyword', + }, + constant_keyword_field: { + type: 'constant_keyword', + }, + date_field: { + type: 'date', + }, + multifield_field: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, + geopoint_field: { + type: 'geo_point', + }, + ip_field: { + type: 'ip', + }, + keyword_field: { + ignore_above: 1024, + type: 'keyword', + }, + long_field: { + type: 'long', + }, + nested_array_field: { + properties: { + field1: { + ignore_above: 1024, + type: 'keyword', + }, + field2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + type: 'nested', + }, + parent_field: { + properties: { + child1: { + ignore_above: 1024, + type: 'keyword', + }, + child2: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + scaled_float_field: { + scaling_factor: 1000, + type: 'scaled_float', + }, + unmapped_object: { + enabled: false, + type: 'object', + }, + formatted_field: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + }, +}; + +describe('mappingFromFieldMap', () => { + it('correctly creates mapping from field map', () => { + expect(mappingFromFieldMap(testFieldMap)).toEqual({ + dynamic: 'strict', + ...expectedTestMapping, + }); + expect(mappingFromFieldMap(alertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + '@timestamp': { + ignore_malformed: false, + type: 'date', + }, + event: { + properties: { + action: { + type: 'keyword', + }, + kind: { + type: 'keyword', + }, + }, + }, + kibana: { + properties: { + alert: { + properties: { + action_group: { + type: 'keyword', + }, + case_ids: { + type: 'keyword', + }, + duration: { + properties: { + us: { + type: 'long', + }, + }, + }, + end: { + type: 'date', + }, + flapping: { + type: 'boolean', + }, + flapping_history: { + type: 'boolean', + }, + maintenance_window_ids: { + type: 'keyword', + }, + instance: { + properties: { + id: { + type: 'keyword', + }, + }, + }, + last_detected: { + type: 'date', + }, + reason: { + fields: { + text: { + type: 'match_only_text', + }, + }, + type: 'keyword', + }, + rule: { + properties: { + category: { + type: 'keyword', + }, + consumer: { + type: 'keyword', + }, + execution: { + properties: { + uuid: { + type: 'keyword', + }, + }, + }, + name: { + type: 'keyword', + }, + parameters: { + type: 'flattened', + ignore_above: 4096, + }, + producer: { + type: 'keyword', + }, + revision: { + type: 'long', + }, + rule_type_id: { + type: 'keyword', + }, + tags: { + type: 'keyword', + }, + uuid: { + type: 'keyword', + }, + }, + }, + start: { + type: 'date', + }, + status: { + type: 'keyword', + }, + time_range: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + }, + url: { + ignore_above: 2048, + index: false, + type: 'keyword', + }, + uuid: { + type: 'keyword', + }, + workflow_assignee_ids: { + type: 'keyword', + }, + workflow_status: { + type: 'keyword', + }, + workflow_tags: { + type: 'keyword', + }, + }, + }, + space_ids: { + type: 'keyword', + }, + version: { + type: 'version', + }, + }, + }, + tags: { + type: 'keyword', + }, + }, + }); + expect(mappingFromFieldMap(legacyAlertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + kibana: { + properties: { + alert: { + properties: { + risk_score: { type: 'float' }, + rule: { + properties: { + author: { type: 'keyword' }, + created_at: { type: 'date' }, + created_by: { type: 'keyword' }, + description: { type: 'keyword' }, + enabled: { type: 'keyword' }, + from: { type: 'keyword' }, + interval: { type: 'keyword' }, + license: { type: 'keyword' }, + note: { type: 'keyword' }, + references: { type: 'keyword' }, + rule_id: { type: 'keyword' }, + rule_name_override: { type: 'keyword' }, + to: { type: 'keyword' }, + type: { type: 'keyword' }, + updated_at: { type: 'date' }, + updated_by: { type: 'keyword' }, + version: { type: 'keyword' }, + }, + }, + severity: { type: 'keyword' }, + suppression: { + properties: { + docs_count: { type: 'long' }, + end: { type: 'date' }, + terms: { + properties: { field: { type: 'keyword' }, value: { type: 'keyword' } }, + }, + start: { type: 'date' }, + }, + }, + system_status: { type: 'keyword' }, + workflow_reason: { type: 'keyword' }, + workflow_status_updated_at: { type: 'date' }, + workflow_user: { type: 'keyword' }, + }, + }, + }, + }, + ecs: { properties: { version: { type: 'keyword' } } }, + }, + }); + }); + + it('uses dynamic setting if specified', () => { + expect(mappingFromFieldMap(testFieldMap, true)).toEqual({ + dynamic: true, + ...expectedTestMapping, + }); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts new file mode 100644 index 00000000000000..5878cedd441956 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/mapping_from_field_map.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { set } from '@kbn/safer-lodash-set'; +import type { FieldMap, MultiField } from './types'; + +export function mappingFromFieldMap( + fieldMap: FieldMap, + dynamic: 'strict' | boolean = 'strict' +): MappingTypeMapping { + const mappings = { + dynamic, + properties: {}, + }; + + const fields = Object.keys(fieldMap).map((key: string) => { + const field = fieldMap[key]; + return { + name: key, + ...field, + }; + }); + + fields.forEach((field) => { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { name, required, array, multi_fields, ...rest } = field; + const mapped = multi_fields + ? { + ...rest, + // eslint-disable-next-line @typescript-eslint/naming-convention + fields: multi_fields.reduce((acc, multi_field: MultiField) => { + acc[multi_field.name] = { + type: multi_field.type, + }; + return acc; + }, {} as Record), + } + : rest; + + set(mappings.properties, field.name.split('.').join('.properties.'), mapped); + + if (name === '@timestamp') { + set(mappings.properties, `${name}.ignore_malformed`, false); + } + }); + + return mappings; +} diff --git a/packages/kbn-data-stream-adapter/src/field_maps/types.ts b/packages/kbn-data-stream-adapter/src/field_maps/types.ts new file mode 100644 index 00000000000000..0a0b68a2f26e6b --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/field_maps/types.ts @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface AllowedValue { + description?: string; + name?: string; +} + +export interface MultiField { + flat_name: string; + name: string; + type: string; +} + +export interface EcsMetadata { + allowed_values?: AllowedValue[]; + dashed_name: string; + description: string; + doc_values?: boolean; + example?: string | number | boolean; + flat_name: string; + ignore_above?: number; + index?: boolean; + level: string; + multi_fields?: MultiField[]; + name: string; + normalize: string[]; + required?: boolean; + scaling_factor?: number; + short: string; + type: string; + properties?: Record; +} + +export interface FieldMap { + [key: string]: { + type: string; + required: boolean; + array?: boolean; + doc_values?: boolean; + enabled?: boolean; + format?: string; + ignore_above?: number; + multi_fields?: MultiField[]; + index?: boolean; + path?: string; + scaling_factor?: number; + dynamic?: boolean | 'strict'; + properties?: Record; + }; +} diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts b/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts new file mode 100644 index 00000000000000..59945b23124c6e --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/install_with_timeout.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { loggerMock } from '@kbn/logging-mocks'; + +import { installWithTimeout } from './install_with_timeout'; +import { ReplaySubject, type Subject } from 'rxjs'; + +const logger = loggerMock.create(); + +describe('installWithTimeout', () => { + let pluginStop$: Subject; + + beforeEach(() => { + jest.resetAllMocks(); + pluginStop$ = new ReplaySubject(1); + }); + + it(`should call installFn`, async () => { + const installFn = jest.fn(); + await installWithTimeout({ + installFn, + pluginStop$, + timeoutMs: 10, + }); + expect(installFn).toHaveBeenCalled(); + }); + + it(`should short-circuit installFn if it exceeds configured timeout`, async () => { + await expect(() => + installWithTimeout({ + installFn: async () => { + await new Promise((r) => setTimeout(r, 20)); + }, + pluginStop$, + timeoutMs: 10, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Failure during installation. Timeout: it took more than 10ms"` + ); + }); + + it(`should short-circuit installFn if pluginStop$ signal is received`, async () => { + pluginStop$.next(); + await expect(() => + installWithTimeout({ + installFn: async () => { + await new Promise((r) => setTimeout(r, 5)); + logger.info(`running`); + }, + pluginStop$, + timeoutMs: 10, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Server is stopping; must stop all async operations"` + ); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/install_with_timeout.ts b/packages/kbn-data-stream-adapter/src/install_with_timeout.ts new file mode 100644 index 00000000000000..7995fed5152ad7 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/install_with_timeout.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { firstValueFrom, type Observable } from 'rxjs'; + +const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes + +interface InstallWithTimeoutOpts { + description?: string; + installFn: () => Promise; + pluginStop$: Observable; + timeoutMs?: number; +} + +export class InstallShutdownError extends Error { + constructor() { + super('Server is stopping; must stop all async operations'); + Object.setPrototypeOf(this, InstallShutdownError.prototype); + } +} + +export const installWithTimeout = async ({ + description, + installFn, + pluginStop$, + timeoutMs = INSTALLATION_TIMEOUT, +}: InstallWithTimeoutOpts): Promise => { + try { + let timeoutId: NodeJS.Timeout; + const install = async (): Promise => { + await installFn(); + if (timeoutId) { + clearTimeout(timeoutId); + } + }; + + const throwTimeoutException = (): Promise => { + return new Promise((_, reject) => { + timeoutId = setTimeout(() => { + const msg = `Timeout: it took more than ${timeoutMs}ms`; + reject(new Error(msg)); + }, timeoutMs); + + firstValueFrom(pluginStop$).then(() => { + clearTimeout(timeoutId); + reject(new InstallShutdownError()); + }); + }); + }; + + await Promise.race([install(), throwTimeoutException()]); + } catch (e) { + if (e instanceof InstallShutdownError) { + throw e; + } else { + const reason = e?.message || 'Unknown reason'; + throw new Error( + `Failure during installation${description ? ` of ${description}` : ''}. ${reason}` + ); + } + } +}; diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts b/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts new file mode 100644 index 00000000000000..e53eb7704a06a6 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/resource_installer_utils.test.ts @@ -0,0 +1,170 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getIndexTemplate, getComponentTemplate } from './resource_installer_utils'; + +describe('getIndexTemplate', () => { + const defaultParams = { + name: 'indexTemplateName', + kibanaVersion: '8.12.1', + indexPatterns: ['indexPattern1', 'indexPattern2'], + componentTemplateRefs: ['template1', 'template2'], + totalFieldsLimit: 2500, + }; + + it('should create index template with given parameters and defaults', () => { + const indexTemplate = getIndexTemplate(defaultParams); + + expect(indexTemplate).toEqual({ + name: defaultParams.name, + body: { + data_stream: { hidden: true }, + index_patterns: defaultParams.indexPatterns, + composed_of: defaultParams.componentTemplateRefs, + template: { + settings: { + hidden: true, + auto_expand_replicas: '0-1', + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit, + }, + mappings: { + dynamic: false, + _meta: { + kibana: { + version: defaultParams.kibanaVersion, + }, + managed: true, + namespace: 'default', + }, + }, + }, + _meta: { + kibana: { + version: defaultParams.kibanaVersion, + }, + managed: true, + namespace: 'default', + }, + priority: 7, + }, + }); + }); + + it('should create not hidden index template', () => { + const { body } = getIndexTemplate({ ...defaultParams, hidden: false }); + expect(body?.data_stream?.hidden).toEqual(false); + expect(body?.template?.settings?.hidden).toEqual(false); + }); + + it('should create index template with custom namespace', () => { + const { body } = getIndexTemplate({ ...defaultParams, namespace: 'custom-namespace' }); + expect(body?._meta?.namespace).toEqual('custom-namespace'); + expect(body?.priority).toEqual(16); + }); + + it('should create index template with template overrides', () => { + const { body } = getIndexTemplate({ + ...defaultParams, + template: { + settings: { + number_of_shards: 1, + }, + mappings: { + dynamic: true, + }, + lifecycle: { + data_retention: '30d', + }, + }, + }); + + expect(body?.template?.settings).toEqual({ + hidden: true, + auto_expand_replicas: '0-1', + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': defaultParams.totalFieldsLimit, + number_of_shards: 1, + }); + + expect(body?.template?.mappings).toEqual({ + dynamic: true, + _meta: { + kibana: { + version: defaultParams.kibanaVersion, + }, + managed: true, + namespace: 'default', + }, + }); + + expect(body?.template?.lifecycle).toEqual({ + data_retention: '30d', + }); + }); +}); + +describe('getComponentTemplate', () => { + const defaultParams = { + name: 'componentTemplateName', + kibanaVersion: '8.12.1', + fieldMap: { + field1: { type: 'text', required: true }, + field2: { type: 'keyword', required: false }, + }, + }; + + it('should create component template with given parameters and defaults', () => { + const componentTemplate = getComponentTemplate(defaultParams); + + expect(componentTemplate).toEqual({ + name: defaultParams.name, + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': 1500, + }, + mappings: { + dynamic: 'strict', + properties: { + field1: { + type: 'text', + }, + field2: { + type: 'keyword', + }, + }, + }, + }, + }); + }); + + it('should create component template with custom settings', () => { + const { template } = getComponentTemplate({ + ...defaultParams, + settings: { + number_of_shards: 1, + number_of_replicas: 1, + }, + }); + + expect(template.settings).toEqual({ + number_of_shards: 1, + number_of_replicas: 1, + 'index.mapping.total_fields.limit': 1500, + }); + }); + + it('should create component template with custom dynamic', () => { + const { template } = getComponentTemplate({ ...defaultParams, dynamic: true }); + expect(template.mappings?.dynamic).toEqual(true); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts b/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts new file mode 100644 index 00000000000000..456be9ad8e86f7 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/resource_installer_utils.ts @@ -0,0 +1,106 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { + IndicesPutIndexTemplateRequest, + Metadata, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + ClusterPutComponentTemplateRequest, + IndicesIndexSettings, + IndicesPutIndexTemplateIndexTemplateMapping, +} from '@elastic/elasticsearch/lib/api/types'; +import type { FieldMap } from './field_maps/types'; +import { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; + +interface GetComponentTemplateOpts { + name: string; + fieldMap: FieldMap; + settings?: IndicesIndexSettings; + dynamic?: 'strict' | boolean; +} + +export const getComponentTemplate = ({ + name, + fieldMap, + settings, + dynamic = 'strict', +}: GetComponentTemplateOpts): ClusterPutComponentTemplateRequest => ({ + name, + _meta: { + managed: true, + }, + template: { + settings: { + number_of_shards: 1, + 'index.mapping.total_fields.limit': + Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, + ...settings, + }, + mappings: mappingFromFieldMap(fieldMap, dynamic), + }, +}); + +interface GetIndexTemplateOpts { + name: string; + indexPatterns: string[]; + kibanaVersion: string; + totalFieldsLimit: number; + componentTemplateRefs?: string[]; + namespace?: string; + template?: IndicesPutIndexTemplateIndexTemplateMapping; + hidden?: boolean; +} + +export const getIndexTemplate = ({ + name, + indexPatterns, + kibanaVersion, + totalFieldsLimit, + componentTemplateRefs, + namespace = 'default', + template = {}, + hidden = true, +}: GetIndexTemplateOpts): IndicesPutIndexTemplateRequest => { + const indexMetadata: Metadata = { + kibana: { + version: kibanaVersion, + }, + managed: true, + namespace, + }; + + return { + name, + body: { + data_stream: { hidden }, + index_patterns: indexPatterns, + composed_of: componentTemplateRefs, + template: { + ...template, + settings: { + hidden, + auto_expand_replicas: '0-1', + 'index.mapping.ignore_malformed': true, + 'index.mapping.total_fields.limit': totalFieldsLimit, + ...template.settings, + }, + mappings: { + dynamic: false, + _meta: indexMetadata, + ...template.mappings, + }, + }, + _meta: indexMetadata, + + // By setting the priority to namespace.length, we ensure that if one namespace is a prefix of another namespace + // then newly created indices will use the matching template with the *longest* namespace + priority: namespace.length, + }, + }; +}; diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts new file mode 100644 index 00000000000000..f7d6cca8c5a079 --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.test.ts @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { loggingSystemMock } from '@kbn/core/server/mocks'; +import { errors as EsErrors, type DiagnosticResult } from '@elastic/elasticsearch'; +import { retryTransientEsErrors } from './retry_transient_es_errors'; + +const mockLogger = loggingSystemMock.createLogger(); + +// mock setTimeout to avoid waiting in tests and prevent test flakiness +global.setTimeout = jest.fn((cb) => jest.fn(cb())) as unknown as typeof global.setTimeout; + +describe('retryTransientEsErrors', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each([ + { error: new EsErrors.ConnectionError('test error'), errorType: 'ConnectionError' }, + { + error: new EsErrors.NoLivingConnectionsError('test error', {} as DiagnosticResult), + errorType: 'NoLivingConnectionsError', + }, + { error: new EsErrors.TimeoutError('test error'), errorType: 'TimeoutError' }, + { + error: new EsErrors.ResponseError({ statusCode: 503 } as DiagnosticResult), + errorType: 'ResponseError (Unavailable)', + }, + { + error: new EsErrors.ResponseError({ statusCode: 408 } as DiagnosticResult), + errorType: 'ResponseError (RequestTimeout)', + }, + { + error: new EsErrors.ResponseError({ statusCode: 410 } as DiagnosticResult), + errorType: 'ResponseError (Gone)', + }, + ])('should retry $errorType', async ({ error }) => { + const mockFn = jest.fn(); + mockFn.mockRejectedValueOnce(error); + mockFn.mockResolvedValueOnce('success'); + + const result = await retryTransientEsErrors(mockFn, { logger: mockLogger }); + + expect(result).toEqual('success'); + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + expect(mockLogger.error).not.toHaveBeenCalled(); + }); + + it('should throw non-transient errors', async () => { + const error = new EsErrors.ResponseError({ statusCode: 429 } as DiagnosticResult); + const mockFn = jest.fn(); + mockFn.mockRejectedValueOnce(error); + + await expect(retryTransientEsErrors(mockFn, { logger: mockLogger })).rejects.toEqual(error); + + expect(mockFn).toHaveBeenCalledTimes(1); + expect(mockLogger.warn).not.toHaveBeenCalled(); + }); + + it('should throw if max retries exceeded', async () => { + const error = new EsErrors.ConnectionError('test error'); + const mockFn = jest.fn(); + mockFn.mockRejectedValueOnce(error); + mockFn.mockRejectedValueOnce(error); + + await expect( + retryTransientEsErrors(mockFn, { logger: mockLogger, attempt: 2 }) + ).rejects.toEqual(error); + + expect(mockFn).toHaveBeenCalledTimes(2); + expect(mockLogger.warn).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts new file mode 100644 index 00000000000000..3b436298e5c8dc --- /dev/null +++ b/packages/kbn-data-stream-adapter/src/retry_transient_es_errors.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { Logger } from '@kbn/core/server'; +import { errors as EsErrors } from '@elastic/elasticsearch'; + +const MAX_ATTEMPTS = 3; + +const retryResponseStatuses = [ + 503, // ServiceUnavailable + 408, // RequestTimeout + 410, // Gone +]; + +const isRetryableError = (e: Error) => + e instanceof EsErrors.NoLivingConnectionsError || + e instanceof EsErrors.ConnectionError || + e instanceof EsErrors.TimeoutError || + (e instanceof EsErrors.ResponseError && + e?.statusCode && + retryResponseStatuses.includes(e.statusCode)); + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const retryTransientEsErrors = async ( + esCall: () => Promise, + { logger, attempt = 0 }: { logger: Logger; attempt?: number } +): Promise => { + try { + return await esCall(); + } catch (e) { + if (attempt < MAX_ATTEMPTS && isRetryableError(e)) { + const retryCount = attempt + 1; + const retryDelaySec: number = Math.min(Math.pow(2, retryCount), 30); // 2s, 4s, 8s, 16s, 30s, 30s, 30s... + + logger.warn( + `Retrying Elasticsearch operation after [${retryDelaySec}s] due to error: ${e.toString()} ${ + e.stack + }` + ); + + // delay with some randomness + await delay(retryDelaySec * 1000 * Math.random()); + return retryTransientEsErrors(esCall, { logger, attempt: retryCount }); + } + + throw e; + } +}; diff --git a/packages/kbn-data-stream-adapter/tsconfig.json b/packages/kbn-data-stream-adapter/tsconfig.json new file mode 100644 index 00000000000000..f09d2b4354d026 --- /dev/null +++ b/packages/kbn-data-stream-adapter/tsconfig.json @@ -0,0 +1,24 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node", + "react", + "@emotion/react/types/css-prop", + "@testing-library/jest-dom", + "@testing-library/react" + ] + }, + "include": ["**/*.ts", "**/*.tsx"], + "kbn_references": [ + "@kbn/core", + "@kbn/std", + "@kbn/ecs", + "@kbn/alerts-as-data-utils", + "@kbn/safer-lodash-set", + "@kbn/logging-mocks", + ], + "exclude": ["target/**/*"] +} diff --git a/tsconfig.base.json b/tsconfig.base.json index 957f6a68e0cbe0..5760913365447c 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -632,6 +632,8 @@ "@kbn/data-search-plugin/*": ["test/plugin_functional/plugins/data_search/*"], "@kbn/data-service": ["packages/kbn-data-service"], "@kbn/data-service/*": ["packages/kbn-data-service/*"], + "@kbn/data-stream-adapter": ["packages/kbn-data-stream-adapter"], + "@kbn/data-stream-adapter/*": ["packages/kbn-data-stream-adapter/*"], "@kbn/data-view-editor-plugin": ["src/plugins/data_view_editor"], "@kbn/data-view-editor-plugin/*": ["src/plugins/data_view_editor/*"], "@kbn/data-view-field-editor-example-plugin": ["examples/data_view_field_editor_example"], diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.test.tsx index 26843c67ff64e7..6bf41d9f18b7f3 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.test.tsx @@ -5,6 +5,7 @@ * 2.0. */ +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; import { renderHook } from '@testing-library/react-hooks'; import React from 'react'; @@ -13,6 +14,7 @@ import { DataQualityProvider, useDataQualityContext } from '.'; const mockReportDataQualityIndexChecked = jest.fn(); const mockReportDataQualityCheckAllClicked = jest.fn(); const mockHttpFetch = jest.fn(); +const { toasts } = notificationServiceMock.createSetupContract(); const mockTelemetryEvents = { reportDataQualityIndexChecked: mockReportDataQualityIndexChecked, reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked, @@ -22,6 +24,7 @@ const ContextWrapper: React.FC = ({ children }) => ( httpFetch={mockHttpFetch} telemetryEvents={mockTelemetryEvents} isILMAvailable={true} + toasts={toasts} > {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.tsx index 4f5c883ddd6968..41175a6793adb2 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/data_quality_context/index.tsx @@ -5,14 +5,16 @@ * 2.0. */ -import type { HttpHandler } from '@kbn/core-http-browser'; import React, { useMemo } from 'react'; -import { TelemetryEvents } from '../../types'; +import type { HttpHandler } from '@kbn/core-http-browser'; +import type { IToasts } from '@kbn/core-notifications-browser'; +import type { TelemetryEvents } from '../../types'; interface DataQualityProviderProps { httpFetch: HttpHandler; isILMAvailable: boolean; telemetryEvents: TelemetryEvents; + toasts: IToasts; } const DataQualityContext = React.createContext(undefined); @@ -20,16 +22,18 @@ const DataQualityContext = React.createContext = ({ children, httpFetch, + toasts, isILMAvailable, telemetryEvents, }) => { const value = useMemo( () => ({ httpFetch, + toasts, isILMAvailable, telemetryEvents, }), - [httpFetch, isILMAvailable, telemetryEvents] + [httpFetch, toasts, isILMAvailable, telemetryEvents] ); return {children}; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx index 2251d0c0fdb230..3de5aba8bcc59d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx @@ -44,7 +44,7 @@ import { useAddToNewCase } from '../../use_add_to_new_case'; import { useMappings } from '../../use_mappings'; import { useUnallowedValues } from '../../use_unallowed_values'; import { useDataQualityContext } from '../data_quality_context'; -import { getSizeInBytes } from '../../helpers'; +import { getSizeInBytes, postResult } from '../../helpers'; const EMPTY_MARKDOWN_COMMENTS: string[] = []; @@ -104,7 +104,7 @@ const IndexPropertiesComponent: React.FC = ({ updatePatternRollup, }) => { const { error: mappingsError, indexes, loading: loadingMappings } = useMappings(indexName); - const { telemetryEvents, isILMAvailable } = useDataQualityContext(); + const { telemetryEvents, isILMAvailable, httpFetch, toasts } = useDataQualityContext(); const requestItems = useMemo( () => @@ -249,7 +249,7 @@ const IndexPropertiesComponent: React.FC = ({ }) : EMPTY_MARKDOWN_COMMENTS; - updatePatternRollup({ + const updatedRollup = { ...patternRollup, results: { ...patternRollup.results, @@ -264,10 +264,11 @@ const IndexPropertiesComponent: React.FC = ({ sameFamily: indexSameFamily, }, }, - }); + }; + updatePatternRollup(updatedRollup); if (indexId && requestTime != null && requestTime > 0 && partitionedFieldMetadata) { - telemetryEvents.reportDataQualityIndexChecked?.({ + const checkMetadata = { batchId: uuidv4(), ecsVersion: EcsVersion, errorCount: error ? 1 : 0, @@ -276,7 +277,10 @@ const IndexPropertiesComponent: React.FC = ({ indexName, isCheckAll: false, numberOfDocuments: docsCount, + numberOfFields: partitionedFieldMetadata.all.length, numberOfIncompatibleFields: indexIncompatible, + numberOfEcsFields: partitionedFieldMetadata.ecsCompliant.length, + numberOfCustomFields: partitionedFieldMetadata.custom.length, numberOfIndices: 1, numberOfIndicesChecked: 1, numberOfSameFamily: indexSameFamily, @@ -289,7 +293,11 @@ const IndexPropertiesComponent: React.FC = ({ unallowedValueFields: getIncompatibleValuesFields( partitionedFieldMetadata.incompatible ), - }); + }; + telemetryEvents.reportDataQualityIndexChecked?.(checkMetadata); + + const result = { meta: checkMetadata, rollup: updatedRollup }; + postResult({ result, httpFetch, toasts, abortController: new AbortController() }); } } } @@ -297,6 +305,7 @@ const IndexPropertiesComponent: React.FC = ({ docsCount, formatBytes, formatNumber, + httpFetch, ilmPhase, indexId, indexName, @@ -309,6 +318,7 @@ const IndexPropertiesComponent: React.FC = ({ patternRollup, requestTime, telemetryEvents, + toasts, unallowedValuesError, updatePatternRollup, ]); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts index f3f1c443786151..a20c8144c57738 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts @@ -35,6 +35,9 @@ import { getTotalSizeInBytes, hasValidTimestampMapping, isMappingCompatible, + postResult, + getResults, + ResultData, } from './helpers'; import { hostNameWithTextMapping, @@ -77,6 +80,8 @@ import { PatternRollup, UnallowedValueCount, } from './types'; +import { httpServiceMock } from '@kbn/core-http-browser-mocks'; +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; const ecsMetadata: Record = EcsFlat as unknown as Record; @@ -1489,4 +1494,81 @@ describe('helpers', () => { ]); }); }); + + describe('postResult', () => { + const { fetch } = httpServiceMock.createStartContract(); + const { toasts } = notificationServiceMock.createStartContract(); + beforeEach(() => { + fetch.mockClear(); + }); + + test('it posts the result', async () => { + const result = { meta: {}, rollup: {} } as unknown as ResultData; + await postResult({ + httpFetch: fetch, + result, + abortController: new AbortController(), + toasts, + }); + + expect(fetch).toHaveBeenCalledWith( + '/internal/ecs_data_quality_dashboard/results', + expect.objectContaining({ + method: 'POST', + body: JSON.stringify(result), + }) + ); + }); + + test('it throws error', async () => { + const result = { meta: {}, rollup: {} } as unknown as ResultData; + fetch.mockRejectedValueOnce('test-error'); + await postResult({ + httpFetch: fetch, + result, + abortController: new AbortController(), + toasts, + }); + expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) }); + }); + }); + + describe('getResults', () => { + const { fetch } = httpServiceMock.createStartContract(); + const { toasts } = notificationServiceMock.createStartContract(); + beforeEach(() => { + fetch.mockClear(); + }); + + test('it gets the results', async () => { + await getResults({ + httpFetch: fetch, + abortController: new AbortController(), + patterns: ['auditbeat-*', 'packetbeat-*'], + toasts, + }); + + expect(fetch).toHaveBeenCalledWith( + '/internal/ecs_data_quality_dashboard/results', + expect.objectContaining({ + method: 'GET', + query: { patterns: 'auditbeat-*,packetbeat-*' }, + }) + ); + }); + + it('should catch error', async () => { + fetch.mockRejectedValueOnce('test-error'); + + const results = await getResults({ + httpFetch: fetch, + abortController: new AbortController(), + patterns: ['auditbeat-*', 'packetbeat-*'], + toasts, + }); + + expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) }); + expect(results).toEqual([]); + }); + }); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts index ba195f0de0e159..a4d51233232e46 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts @@ -5,11 +5,13 @@ * 2.0. */ +import type { HttpHandler } from '@kbn/core-http-browser'; import type { IlmExplainLifecycleLifecycleExplain, IndicesStatsIndicesStats, } from '@elastic/elasticsearch/lib/api/types'; import { has, sortBy } from 'lodash/fp'; +import { IToasts } from '@kbn/core-notifications-browser'; import { getIlmPhase } from './data_quality_panel/pattern/helpers'; import { getFillColor } from './data_quality_panel/tabs/summary_tab/helpers'; @@ -17,6 +19,7 @@ import * as i18n from './translations'; import type { DataQualityCheckResult, + DataQualityIndexCheckedParams, EcsMetadata, EnrichedFieldMetadata, ErrorSummary, @@ -443,3 +446,58 @@ export const getErrorSummaries = ( [] ); }; + +export const RESULTS_API_ROUTE = '/internal/ecs_data_quality_dashboard/results'; + +export interface ResultData { + meta: DataQualityIndexCheckedParams; + rollup: PatternRollup; +} + +export async function postResult({ + result, + httpFetch, + toasts, + abortController, +}: { + result: ResultData; + httpFetch: HttpHandler; + toasts: IToasts; + abortController: AbortController; +}): Promise { + try { + await httpFetch(RESULTS_API_ROUTE, { + method: 'POST', + signal: abortController.signal, + version: INTERNAL_API_VERSION, + body: JSON.stringify(result), + }); + } catch (err) { + toasts.addError(err, { title: i18n.POST_RESULT_ERROR_TITLE }); + } +} + +export async function getResults({ + patterns, + httpFetch, + toasts, + abortController, +}: { + patterns: string[]; + httpFetch: HttpHandler; + toasts: IToasts; + abortController: AbortController; +}): Promise { + try { + const results = await httpFetch(RESULTS_API_ROUTE, { + method: 'GET', + signal: abortController.signal, + version: INTERNAL_API_VERSION, + query: { patterns: patterns.join(',') }, + }); + return results; + } catch (err) { + toasts.addError(err, { title: i18n.GET_RESULTS_ERROR_TITLE }); + return []; + } +} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.test.tsx index e7e72221d18f59..720f2fc61da6c1 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.test.tsx @@ -11,6 +11,9 @@ import React from 'react'; import { TestProviders } from './mock/test_providers/test_providers'; import { DataQualityPanel } from '.'; +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; + +const { toasts } = notificationServiceMock.createSetupContract(); describe('DataQualityPanel', () => { describe('when ILM phases are provided', () => { @@ -20,7 +23,6 @@ describe('DataQualityPanel', () => { render( { reportDataQualityIndexChecked={jest.fn()} setLastChecked={jest.fn()} baseTheme={DARK_THEME} + toasts={toasts} /> ); @@ -56,7 +59,6 @@ describe('DataQualityPanel', () => { render( { reportDataQualityIndexChecked={jest.fn()} setLastChecked={jest.fn()} baseTheme={DARK_THEME} + toasts={toasts} /> ); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.tsx index 8d95e714647287..6db2d8991db827 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/index.tsx @@ -19,13 +19,14 @@ import type { } from '@elastic/charts'; import React, { useCallback, useMemo } from 'react'; +import type { IToasts } from '@kbn/core-notifications-browser'; import { Body } from './data_quality_panel/body'; import { DataQualityProvider } from './data_quality_panel/data_quality_context'; import { EMPTY_STAT } from './helpers'; import { ReportDataQualityCheckAllCompleted, ReportDataQualityIndexChecked } from './types'; interface Props { - addSuccessToast: (toast: { title: string }) => void; + toasts: IToasts; baseTheme: Theme; canUserCreateAndReadCases: () => boolean; defaultNumberFormat: string; @@ -66,7 +67,7 @@ interface Props { /** Renders the `Data Quality` dashboard content */ const DataQualityPanelComponent: React.FC = ({ - addSuccessToast, + toasts, baseTheme, canUserCreateAndReadCases, defaultBytesFormat, @@ -103,11 +104,19 @@ const DataQualityPanelComponent: React.FC = ({ [reportDataQualityCheckAllCompleted, reportDataQualityIndexChecked] ); + const addSuccessToast = useCallback( + (toast: { title: string }) => { + toasts.addSuccess(toast); + }, + [toasts] + ); + return ( = ({ children, isILMAvailable = true }) => { const http = httpServiceMock.createSetupContract({ basePath: '/test' }); + const { toasts } = notificationServiceMock.createSetupContract(); const actionTypeRegistry = actionTypeRegistryMock.create(); const mockGetInitialConversations = jest.fn(() => ({})); const mockGetComments = jest.fn(() => []); @@ -79,6 +81,7 @@ export const TestProvidersComponent: React.FC = ({ children, isILMAvailab > diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/translations.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/translations.ts index 99d94c73ff49c6..b72a9fee96c574 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/translations.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/translations.ts @@ -292,3 +292,13 @@ export const WARM_PATTERN_TOOLTIP = ({ indices, pattern }: { indices: number; pa defaultMessage: '{indices} {indices, plural, =1 {index} other {indices}} matching the {pattern} pattern {indices, plural, =1 {is} other {are}} warm. Warm indices are no longer being updated but are still being queried.', }); + +export const POST_RESULT_ERROR_TITLE = i18n.translate( + 'securitySolutionPackages.ecsDataQualityDashboard.postResultErrorTitle', + { defaultMessage: 'Error writing saved data quality check results' } +); + +export const GET_RESULTS_ERROR_TITLE = i18n.translate( + 'securitySolutionPackages.ecsDataQualityDashboard.getResultErrorTitle', + { defaultMessage: 'Error reading saved data quality check results' } +); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts index 3fa021a9b3690d..9f507992d15093 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts @@ -142,19 +142,7 @@ export interface IndexToCheck { indexName: string; } -export type OnCheckCompleted = ({ - batchId, - checkAllStartTime, - error, - formatBytes, - formatNumber, - indexName, - isLastCheck, - partitionedFieldMetadata, - pattern, - version, - requestTime, -}: { +export type OnCheckCompleted = (param: { batchId: string; checkAllStartTime: number; error: string | null; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.test.tsx index 90cd0906c137a2..f715936501736e 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_ilm_explain/index.test.tsx @@ -12,6 +12,7 @@ import { DataQualityProvider } from '../data_quality_panel/data_quality_context' import { mockIlmExplain } from '../mock/ilm_explain/mock_ilm_explain'; import { ERROR_LOADING_ILM_EXPLAIN } from '../translations'; import { useIlmExplain, UseIlmExplain } from '.'; +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; const mockHttpFetch = jest.fn(); const mockReportDataQualityIndexChecked = jest.fn(); @@ -20,6 +21,7 @@ const mockTelemetryEvents = { reportDataQualityIndexChecked: mockReportDataQualityIndexChecked, reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked, }; +const { toasts } = notificationServiceMock.createSetupContract(); const ContextWrapper: React.FC<{ children: React.ReactNode; isILMAvailable: boolean }> = ({ children, isILMAvailable = true, @@ -28,6 +30,7 @@ const ContextWrapper: React.FC<{ children: React.ReactNode; isILMAvailable: bool httpFetch={mockHttpFetch} telemetryEvents={mockTelemetryEvents} isILMAvailable={isILMAvailable} + toasts={toasts} > {children} @@ -76,6 +79,7 @@ describe('useIlmExplain', () => { httpFetch={mockHttpFetch} telemetryEvents={mockTelemetryEvents} isILMAvailable={false} + toasts={toasts} > {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/index.test.tsx index 06006d3c5a3cfe..b7e0854b46443d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_mappings/index.test.tsx @@ -12,6 +12,7 @@ import { DataQualityProvider } from '../data_quality_panel/data_quality_context' import { mockMappingsResponse } from '../mock/mappings_response/mock_mappings_response'; import { ERROR_LOADING_MAPPINGS } from '../translations'; import { useMappings, UseMappings } from '.'; +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; const mockHttpFetch = jest.fn(); const mockReportDataQualityIndexChecked = jest.fn(); @@ -20,12 +21,14 @@ const mockTelemetryEvents = { reportDataQualityIndexChecked: mockReportDataQualityIndexChecked, reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked, }; +const { toasts } = notificationServiceMock.createSetupContract(); const ContextWrapper: React.FC = ({ children }) => ( {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts index 9175acf5061b15..b37d0aecc25cfe 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts @@ -13,7 +13,6 @@ import { getTotalIndices, getTotalIndicesChecked, getTotalSameFamily, - onPatternRollupUpdated, updateResultOnCheckCompleted, } from './helpers'; import { auditbeatWithAllResults } from '../mock/pattern_rollup/mock_auditbeat_pattern_rollup'; @@ -166,21 +165,6 @@ describe('helpers', () => { }); }); - describe('onPatternRollupUpdated', () => { - test('it returns a new collection with the updated rollup', () => { - const before: Record = { - 'auditbeat-*': auditbeatWithAllResults, - }; - - expect( - onPatternRollupUpdated({ - patternRollup: mockPacketbeatPatternRollup, - patternRollups: before, - }) - ).toEqual(patternRollups); - }); - }); - describe('updateResultOnCheckCompleted', () => { const packetbeatStats861: IndicesStatsIndicesStats = mockPacketbeatPatternRollup.stats != null diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts index b20dfcf6648bc7..cca8ac331aa885 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts @@ -87,17 +87,6 @@ export const getTotalIndicesChecked = (patternRollups: Record; -}): Record => ({ - ...patternRollups, - [patternRollup.pattern]: patternRollup, -}); - export const updateResultOnCheckCompleted = ({ error, formatBytes, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx index 03f5035868b19f..27810fedffde4f 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx @@ -8,6 +8,7 @@ import { useCallback, useEffect, useMemo, useState } from 'react'; import { EcsVersion } from '@kbn/ecs'; +import { isEmpty } from 'lodash/fp'; import { getTotalDocsCount, getTotalIncompatible, @@ -15,12 +16,18 @@ import { getTotalIndicesChecked, getTotalSameFamily, getTotalSizeInBytes, - onPatternRollupUpdated, updateResultOnCheckCompleted, } from './helpers'; import type { OnCheckCompleted, PatternRollup } from '../types'; -import { getDocsCount, getIndexId, getSizeInBytes, getTotalPatternSameFamily } from '../helpers'; +import { + getDocsCount, + getIndexId, + getResults, + getSizeInBytes, + getTotalPatternSameFamily, + postResult, +} from '../helpers'; import { getIlmPhase, getIndexIncompatible } from '../data_quality_panel/pattern/helpers'; import { useDataQualityContext } from '../data_quality_panel/data_quality_context'; import { @@ -53,15 +60,60 @@ interface UseResultsRollup { updatePatternRollup: (patternRollup: PatternRollup) => void; } +const useStoredPatternRollups = (patterns: string[]) => { + const { httpFetch, toasts } = useDataQualityContext(); + const [storedRollups, setStoredRollups] = useState>({}); + + useEffect(() => { + if (isEmpty(patterns)) { + return; + } + + let ignore = false; + const abortController = new AbortController(); + const fetchStoredRollups = async () => { + const results = await getResults({ httpFetch, abortController, patterns, toasts }); + if (results?.length && !ignore) { + setStoredRollups(Object.fromEntries(results.map(({ rollup }) => [rollup.pattern, rollup]))); + } + }; + + fetchStoredRollups(); + return () => { + ignore = true; + }; + }, [httpFetch, patterns, toasts]); + + return storedRollups; +}; + export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRollup => { + const { httpFetch, toasts } = useDataQualityContext(); const [patternIndexNames, setPatternIndexNames] = useState>({}); const [patternRollups, setPatternRollups] = useState>({}); + + const storedPatternsRollups = useStoredPatternRollups(patterns); + + useEffect(() => { + if (!isEmpty(storedPatternsRollups)) { + setPatternRollups((current) => ({ ...current, ...storedPatternsRollups })); + } + }, [storedPatternsRollups]); + + const updatePatternRollups = useCallback( + (updateRollups: (current: Record) => Record) => { + setPatternRollups((current) => updateRollups(current)); + }, + [] + ); + const { telemetryEvents, isILMAvailable } = useDataQualityContext(); - const updatePatternRollup = useCallback((patternRollup: PatternRollup) => { - setPatternRollups((current) => - onPatternRollupUpdated({ patternRollup, patternRollups: current }) - ); - }, []); + const updatePatternRollup = useCallback( + (patternRollup: PatternRollup) => { + updatePatternRollups((current) => ({ ...current, [patternRollup.pattern]: patternRollup })); + }, + [updatePatternRollups] + ); const totalDocsCount = useMemo(() => getTotalDocsCount(patternRollups), [patternRollups]); const totalIncompatible = useMemo(() => getTotalIncompatible(patternRollups), [patternRollups]); @@ -75,10 +127,7 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll const updatePatternIndexNames = useCallback( ({ indexNames, pattern }: { indexNames: string[]; pattern: string }) => { - setPatternIndexNames((current) => ({ - ...current, - [pattern]: indexNames, - })); + setPatternIndexNames((current) => ({ ...current, [pattern]: indexNames })); }, [] ); @@ -96,11 +145,8 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll requestTime, isLastCheck, }) => { - const indexId = getIndexId({ indexName, stats: patternRollups[pattern].stats }); - const ilmExplain = patternRollups[pattern].ilmExplain; - - setPatternRollups((current) => { - const updated = updateResultOnCheckCompleted({ + setPatternRollups((currentPatternRollups) => { + const updatedRollups = updateResultOnCheckCompleted({ error, formatBytes, formatNumber, @@ -108,19 +154,23 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll isILMAvailable, partitionedFieldMetadata, pattern, - patternRollups: current, + patternRollups: currentPatternRollups, }); + const updatedRollup = updatedRollups[pattern]; + const { stats, results, ilmExplain } = updatedRollup; + const indexId = getIndexId({ indexName, stats }); + if ( indexId != null && - updated[pattern].stats && - updated[pattern].results && + stats && + results && + ilmExplain && requestTime != null && requestTime > 0 && - partitionedFieldMetadata && - ilmExplain + partitionedFieldMetadata ) { - telemetryEvents.reportDataQualityIndexChecked?.({ + const metadata = { batchId, ecsVersion: EcsVersion, errorCount: error ? 1 : 0, @@ -128,16 +178,19 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll indexId, indexName, isCheckAll: true, - numberOfDocuments: getDocsCount({ indexName, stats: updated[pattern].stats }), + numberOfDocuments: getDocsCount({ indexName, stats }), + numberOfFields: partitionedFieldMetadata.all.length, numberOfIncompatibleFields: getIndexIncompatible({ indexName, - results: updated[pattern].results, + results, }), + numberOfEcsFields: partitionedFieldMetadata.ecsCompliant.length, + numberOfCustomFields: partitionedFieldMetadata.custom.length, numberOfIndices: 1, numberOfIndicesChecked: 1, - numberOfSameFamily: getTotalPatternSameFamily(updated[pattern].results), + numberOfSameFamily: getTotalPatternSameFamily(results), sameFamilyFields: getSameFamilyFields(partitionedFieldMetadata.sameFamily), - sizeInBytes: getSizeInBytes({ stats: updated[pattern].stats, indexName }), + sizeInBytes: getSizeInBytes({ stats, indexName }), timeConsumedMs: requestTime, unallowedMappingFields: getIncompatibleMappingsFields( partitionedFieldMetadata.incompatible @@ -145,7 +198,11 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll unallowedValueFields: getIncompatibleValuesFields( partitionedFieldMetadata.incompatible ), - }); + }; + telemetryEvents.reportDataQualityIndexChecked?.(metadata); + + const result = { meta: metadata, rollup: updatedRollup }; + postResult({ result, httpFetch, toasts, abortController: new AbortController() }); } if (isLastCheck) { @@ -153,19 +210,19 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll batchId, ecsVersion: EcsVersion, isCheckAll: true, - numberOfDocuments: getTotalDocsCount(updated), - numberOfIncompatibleFields: getTotalIncompatible(updated), - numberOfIndices: getTotalIndices(updated), - numberOfIndicesChecked: getTotalIndicesChecked(updated), - numberOfSameFamily: getTotalSameFamily(updated), - sizeInBytes: getTotalSizeInBytes(updated), + numberOfDocuments: getTotalDocsCount(updatedRollups), + numberOfIncompatibleFields: getTotalIncompatible(updatedRollups), + numberOfIndices: getTotalIndices(updatedRollups), + numberOfIndicesChecked: getTotalIndicesChecked(updatedRollups), + numberOfSameFamily: getTotalSameFamily(updatedRollups), + sizeInBytes: getTotalSizeInBytes(updatedRollups), timeConsumedMs: Date.now() - checkAllStartTime, }); } - return updated; + return updatedRollups; }); }, - [isILMAvailable, patternRollups, telemetryEvents] + [httpFetch, isILMAvailable, telemetryEvents, toasts] ); useEffect(() => { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx index de63f40e361c42..fb13413dbbd4d7 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_stats/index.test.tsx @@ -12,6 +12,7 @@ import { DataQualityProvider } from '../data_quality_panel/data_quality_context' import { mockStatsGreenIndex } from '../mock/stats/mock_stats_green_index'; import { ERROR_LOADING_STATS } from '../translations'; import { useStats, UseStats } from '.'; +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; const mockHttpFetch = jest.fn(); const mockReportDataQualityIndexChecked = jest.fn(); @@ -20,12 +21,14 @@ const mockTelemetryEvents = { reportDataQualityIndexChecked: mockReportDataQualityIndexChecked, reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked, }; +const { toasts } = notificationServiceMock.createSetupContract(); const ContextWrapper: React.FC = ({ children }) => ( {children} @@ -36,6 +39,7 @@ const ContextWrapperILMNotAvailable: React.FC = ({ children }) => ( httpFetch={mockHttpFetch} telemetryEvents={mockTelemetryEvents} isILMAvailable={false} + toasts={toasts} > {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx index b69de24cb6d9ae..f5bfcde8cb3b66 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_unallowed_values/index.test.tsx @@ -15,6 +15,7 @@ import { mockUnallowedValuesResponse } from '../mock/unallowed_values/mock_unall import { ERROR_LOADING_UNALLOWED_VALUES } from '../translations'; import { EcsMetadata, UnallowedValueRequestItem } from '../types'; import { useUnallowedValues, UseUnallowedValues } from '.'; +import { notificationServiceMock } from '@kbn/core-notifications-browser-mocks'; const mockHttpFetch = jest.fn(); const mockReportDataQualityIndexChecked = jest.fn(); @@ -23,12 +24,14 @@ const mockTelemetryEvents = { reportDataQualityIndexChecked: mockReportDataQualityIndexChecked, reportDataQualityCheckAllCompleted: mockReportDataQualityCheckAllClicked, }; +const { toasts } = notificationServiceMock.createSetupContract(); const ContextWrapper: React.FC = ({ children }) => ( {children} diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/tsconfig.json b/x-pack/packages/security-solution/ecs_data_quality_dashboard/tsconfig.json index 4581552f2c5915..d005b921762705 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/tsconfig.json +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/tsconfig.json @@ -25,5 +25,7 @@ "@kbn/elastic-assistant", "@kbn/triggers-actions-ui-plugin", "@kbn/core", + "@kbn/core-notifications-browser", + "@kbn/core-notifications-browser-mocks", ] } diff --git a/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts b/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts index 52c734797f726d..d979d22b12b881 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/common/constants.ts @@ -13,4 +13,5 @@ export const GET_INDEX_STATS = `${BASE_PATH}/stats/{pattern}`; export const GET_INDEX_MAPPINGS = `${BASE_PATH}/mappings/{pattern}`; export const GET_UNALLOWED_FIELD_VALUES = `${BASE_PATH}/unallowed_field_values`; export const GET_ILM_EXPLAIN = `${BASE_PATH}/ilm_explain/{pattern}`; +export const RESULTS_ROUTE_PATH = `${BASE_PATH}/results`; export const INTERNAL_API_VERSION = '1'; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc b/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc index b946a7342a1b07..2650184783066f 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc +++ b/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc @@ -8,7 +8,10 @@ "server": true, "browser": false, "requiredPlugins": [ - "data" + "data", + ], + "optionalPlugins": [ + "spaces", ] } } diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/request_context.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/request_context.ts index 19fb44f7f8bace..eaa90f75c58907 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/request_context.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/__mocks__/request_context.ts @@ -39,6 +39,9 @@ const createAppClientMock = () => ({}); const createRequestContextMock = (clients: MockClients = createMockClients()) => { return { core: clients.core, + dataQualityDashboard: { + getResultsIndexName: jest.fn(() => Promise.resolve('mock_results_index_name')), + }, }; }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts index 032a0930cb40f3..8f7fdead515472 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_available_indices.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; +import type { SearchRequest } from '@elastic/elasticsearch/lib/api/types'; export const getRequestBody = ({ indexPattern, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_unallowed_field_requests.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_unallowed_field_requests.ts index c5b30da92295ee..b09959ab7921b5 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_unallowed_field_requests.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/helpers/get_unallowed_field_requests.ts @@ -5,11 +5,11 @@ * 2.0. */ -import { +import type { MsearchMultisearchHeader, MsearchMultisearchBody, } from '@elastic/elasticsearch/lib/api/types'; -import { AllowedValuesInputs } from '../schemas/get_unallowed_field_values'; +import type { AllowedValuesInputs } from '../schemas/get_unallowed_field_values'; export const getMSearchRequestHeader = (indexName: string): MsearchMultisearchHeader => ({ expand_wildcards: ['open'], diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/index.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/index.ts index bd83becac07b3f..eaea7b463eed7c 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/index.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/index.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { PluginInitializerContext } from '@kbn/core/server'; +import type { PluginInitializerContext } from '@kbn/core/server'; // This exports static code and TypeScript types, // as well as, Kibana Platform `plugin()` initializer. diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_data_stream.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_data_stream.test.ts new file mode 100644 index 00000000000000..7504a4b55c1cb8 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_data_stream.test.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ResultsDataStream } from './results_data_stream'; +import { Subject } from 'rxjs'; +import type { InstallParams } from '@kbn/data-stream-adapter'; +import { DataStreamSpacesAdapter } from '@kbn/data-stream-adapter'; +import { elasticsearchServiceMock } from '@kbn/core-elasticsearch-server-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; + +jest.mock('@kbn/data-stream-adapter'); + +const MockedDataStreamSpacesAdapter = DataStreamSpacesAdapter as unknown as jest.MockedClass< + typeof DataStreamSpacesAdapter +>; + +const esClient = elasticsearchServiceMock.createStart().client.asInternalUser; + +describe('ResultsDataStream', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('constructor', () => { + it('should create DataStreamSpacesAdapter', () => { + new ResultsDataStream({ kibanaVersion: '8.13.0' }); + expect(MockedDataStreamSpacesAdapter).toHaveBeenCalledTimes(1); + }); + + it('should create component templates', () => { + new ResultsDataStream({ kibanaVersion: '8.13.0' }); + const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; + expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: '.kibana-data-quality-dashboard-ecs-mappings' }) + ); + expect(dataStreamSpacesAdapter.setComponentTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: '.kibana-data-quality-dashboard-results-mappings' }) + ); + }); + + it('should create index templates', () => { + new ResultsDataStream({ kibanaVersion: '8.13.0' }); + const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; + expect(dataStreamSpacesAdapter.setIndexTemplate).toHaveBeenCalledWith( + expect.objectContaining({ name: '.kibana-data-quality-dashboard-results-index-template' }) + ); + }); + }); + + describe('install', () => { + it('should install data stream', async () => { + const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' }); + const params: InstallParams = { + esClient, + logger: loggerMock.create(), + pluginStop$: new Subject(), + }; + await resultsDataStream.install(params); + const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; + expect(dataStreamSpacesAdapter.install).toHaveBeenCalledWith(params); + }); + + it('should log error', async () => { + const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' }); + const params: InstallParams = { + esClient, + logger: loggerMock.create(), + pluginStop$: new Subject(), + }; + const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; + const error = new Error('test-error'); + (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); + + await resultsDataStream.install(params); + expect(params.logger.error).toHaveBeenCalledWith(expect.any(String), error); + }); + }); + + describe('installSpace', () => { + it('should install space', async () => { + const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' }); + const params: InstallParams = { + esClient, + logger: loggerMock.create(), + pluginStop$: new Subject(), + }; + const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; + (dataStreamSpacesAdapter.install as jest.Mock).mockResolvedValueOnce(undefined); + + await resultsDataStream.install(params); + await resultsDataStream.installSpace('space1'); + + expect(dataStreamSpacesAdapter.getInstalledSpaceName).toHaveBeenCalledWith('space1'); + expect(dataStreamSpacesAdapter.installSpace).toHaveBeenCalledWith('space1'); + }); + + it('should not install space if install not executed', async () => { + const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' }); + expect(resultsDataStream.installSpace('space1')).rejects.toThrowError(); + }); + + it('should throw error if main install had error', async () => { + const resultsDataStream = new ResultsDataStream({ kibanaVersion: '8.13.0' }); + const params: InstallParams = { + esClient, + logger: loggerMock.create(), + pluginStop$: new Subject(), + }; + const [dataStreamSpacesAdapter] = MockedDataStreamSpacesAdapter.mock.instances; + const error = new Error('test-error'); + (dataStreamSpacesAdapter.install as jest.Mock).mockRejectedValueOnce(error); + await resultsDataStream.install(params); + + expect(resultsDataStream.installSpace('space1')).rejects.toThrowError(error); + }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_data_stream.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_data_stream.ts new file mode 100644 index 00000000000000..30f760f7ccdaf7 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_data_stream.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DataStreamSpacesAdapter, ecsFieldMap, type InstallParams } from '@kbn/data-stream-adapter'; +import { resultsFieldMap } from './results_field_map'; + +const TOTAL_FIELDS_LIMIT = 2500; + +const RESULTS_DATA_STREAM_NAME = '.kibana-data-quality-dashboard-results'; + +const RESULTS_INDEX_TEMPLATE_NAME = '.kibana-data-quality-dashboard-results-index-template'; +const RESULTS_COMPONENT_TEMPLATE_NAME = '.kibana-data-quality-dashboard-results-mappings'; +const ECS_COMPONENT_TEMPLATE_NAME = '.kibana-data-quality-dashboard-ecs-mappings'; + +export class ResultsDataStream { + private readonly dataStream: DataStreamSpacesAdapter; + private installPromise?: Promise; + + constructor({ kibanaVersion }: { kibanaVersion: string }) { + this.dataStream = new DataStreamSpacesAdapter(RESULTS_DATA_STREAM_NAME, { + kibanaVersion, + totalFieldsLimit: TOTAL_FIELDS_LIMIT, + }); + this.dataStream.setComponentTemplate({ + name: ECS_COMPONENT_TEMPLATE_NAME, + fieldMap: ecsFieldMap, + }); + this.dataStream.setComponentTemplate({ + name: RESULTS_COMPONENT_TEMPLATE_NAME, + fieldMap: resultsFieldMap, + }); + + this.dataStream.setIndexTemplate({ + name: RESULTS_INDEX_TEMPLATE_NAME, + componentTemplateRefs: [RESULTS_COMPONENT_TEMPLATE_NAME, ECS_COMPONENT_TEMPLATE_NAME], + }); + } + + async install(params: InstallParams) { + try { + this.installPromise = this.dataStream.install(params); + await this.installPromise; + } catch (err) { + params.logger.error( + `Error installing results data stream. Data quality dashboard persistence may be impacted.- ${err.message}`, + err + ); + } + } + + async installSpace(spaceId: string): Promise { + if (!this.installPromise) { + throw new Error('Results data stream not installed'); + } + // wait for install to complete, may reject if install failed, routes should handle this + await this.installPromise; + let dataStreamName = await this.dataStream.getInstalledSpaceName(spaceId); + if (!dataStreamName) { + dataStreamName = await this.dataStream.installSpace(spaceId); + } + return dataStreamName; + } +} diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts new file mode 100644 index 00000000000000..59f8ade6cb8343 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FieldMap } from '@kbn/data-stream-adapter'; + +export const resultsFieldMap: FieldMap = { + 'meta.batchId': { type: 'keyword', required: true }, + 'meta.ecsVersion': { type: 'keyword', required: true }, + 'meta.errorCount': { type: 'long', required: true }, + 'meta.ilmPhase': { type: 'keyword', required: true }, + 'meta.indexId': { type: 'keyword', required: true }, + 'meta.indexName': { type: 'keyword', required: true }, + 'meta.isCheckAll': { type: 'boolean', required: true }, + 'meta.numberOfDocuments': { type: 'long', required: true }, + 'meta.numberOfFields': { type: 'long', required: true }, + 'meta.numberOfIncompatibleFields': { type: 'long', required: true }, + 'meta.numberOfEcsFields': { type: 'long', required: true }, + 'meta.numberOfCustomFields': { type: 'long', required: true }, + 'meta.numberOfIndices': { type: 'long', required: true }, + 'meta.numberOfIndicesChecked': { type: 'long', required: true }, + 'meta.numberOfSameFamily': { type: 'long', required: true }, + 'meta.sameFamilyFields': { type: 'keyword', required: true, array: true }, + 'meta.sizeInBytes': { type: 'long', required: true }, + 'meta.timeConsumedMs': { type: 'long', required: true }, + 'meta.unallowedMappingFields': { type: 'keyword', required: true, array: true }, + 'meta.unallowedValueFields': { type: 'keyword', required: true, array: true }, + 'rollup.docsCount': { type: 'long', required: true }, + 'rollup.error': { type: 'text', required: false }, + 'rollup.ilmExplainPhaseCounts': { type: 'object', required: false }, + 'rollup.indices': { type: 'long', required: true }, + 'rollup.pattern': { type: 'keyword', required: true }, + 'rollup.sizeInBytes': { type: 'long', required: true }, + 'rollup.ilmExplain': { type: 'object', required: true, array: true }, + 'rollup.stats': { type: 'object', required: true, array: true }, + 'rollup.results': { type: 'object', required: true, array: true }, +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_ilm_explain.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_ilm_explain.ts index 6ee120aa04ae32..751104db52879c 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_ilm_explain.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_ilm_explain.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IlmExplainLifecycleResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { IlmExplainLifecycleResponse } from '@elastic/elasticsearch/lib/api/types'; import type { IScopedClusterClient } from '@kbn/core/server'; export const fetchILMExplain = ( diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_mappings.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_mappings.ts index e0364acbb559e7..d63a315a4d1e28 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_mappings.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_mappings.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesGetMappingIndexMappingRecord } from '@elastic/elasticsearch/lib/api/types'; import type { IScopedClusterClient } from '@kbn/core/server'; export const fetchMappings = ( diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts index b9a6abd7e54db1..c02d5361bdc8e6 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/fetch_stats.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IndicesStatsResponse } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesStatsResponse } from '@elastic/elasticsearch/lib/api/types'; import type { IScopedClusterClient } from '@kbn/core/server'; export const fetchStats = ( diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/get_unallowed_field_values.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/get_unallowed_field_values.ts index 2079a58ce5b8df..e549c275d3d35d 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/get_unallowed_field_values.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/get_unallowed_field_values.ts @@ -6,12 +6,12 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; -import { MsearchRequestItem } from '@elastic/elasticsearch/lib/api/types'; +import type { MsearchRequestItem } from '@elastic/elasticsearch/lib/api/types'; import { getMSearchRequestBody, getMSearchRequestHeader, } from '../helpers/get_unallowed_field_requests'; -import { GetUnallowedFieldValuesInputs } from '../schemas/get_unallowed_field_values'; +import type { GetUnallowedFieldValuesInputs } from '../schemas/get_unallowed_field_values'; export const getUnallowedFieldValues = ( esClient: ElasticsearchClient, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts index 0c1cf336dc10df..19c6f12479694a 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts @@ -5,34 +5,77 @@ * 2.0. */ -import { PluginInitializerContext, CoreSetup, CoreStart, Plugin, Logger } from '@kbn/core/server'; +import type { + PluginInitializerContext, + CoreSetup, + CoreStart, + Plugin, + Logger, +} from '@kbn/core/server'; -import { EcsDataQualityDashboardPluginSetup, EcsDataQualityDashboardPluginStart } from './types'; +import { ReplaySubject, type Subject } from 'rxjs'; +import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; +import type { + EcsDataQualityDashboardPluginSetup, + EcsDataQualityDashboardPluginStart, + PluginSetupDependencies, + DataQualityDashboardRequestHandlerContext, +} from './types'; import { getILMExplainRoute, getIndexMappingsRoute, getIndexStatsRoute, getUnallowedFieldValuesRoute, + resultsRoutes, } from './routes'; +import { ResultsDataStream } from './lib/data_stream/results_data_stream'; export class EcsDataQualityDashboardPlugin implements Plugin { private readonly logger: Logger; + private readonly resultsDataStream: ResultsDataStream; + private pluginStop$: Subject; constructor(initializerContext: PluginInitializerContext) { this.logger = initializerContext.logger.get(); + this.pluginStop$ = new ReplaySubject(1); + this.resultsDataStream = new ResultsDataStream({ + kibanaVersion: initializerContext.env.packageInfo.version, + }); } - public setup(core: CoreSetup) { - this.logger.debug('ecsDataQualityDashboard: Setup'); // this would be deleted when plugin is removed - const router = core.http.createRouter(); // this would be deleted when plugin is removed + public setup(core: CoreSetup, plugins: PluginSetupDependencies) { + this.logger.debug('ecsDataQualityDashboard: Setup'); + + // TODO: Uncomment https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 + // this.resultsDataStream.install({ + // esClient: core + // .getStartServices() + // .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + // logger: this.logger, + // pluginStop$: this.pluginStop$, + // }); + + core.http.registerRouteHandlerContext< + DataQualityDashboardRequestHandlerContext, + 'dataQualityDashboard' + >('dataQualityDashboard', (_context, request) => { + const spaceId = plugins.spaces?.spacesService.getSpaceId(request) ?? DEFAULT_SPACE_ID; + return { + spaceId, + getResultsIndexName: async () => this.resultsDataStream.installSpace(spaceId), + }; + }); + + const router = core.http.createRouter(); // Register server side APIs getIndexMappingsRoute(router, this.logger); getIndexStatsRoute(router, this.logger); getUnallowedFieldValuesRoute(router, this.logger); getILMExplainRoute(router, this.logger); + resultsRoutes(router, this.logger); return {}; } @@ -41,5 +84,8 @@ export class EcsDataQualityDashboardPlugin return {}; } - public stop() {} + public stop() { + this.pluginStop$.next(); + this.pluginStop$.complete(); + } } diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts index 329defab80c2b0..2fa4243f002b92 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.test.ts @@ -12,7 +12,7 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getILMExplainRoute } from './get_ilm_explain'; -import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ fetchILMExplain: jest.fn(), diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts index 86db6f6c79004e..73282d11e3d71e 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IRouter, Logger } from '@kbn/core/server'; +import type { IRouter, Logger } from '@kbn/core/server'; import { GET_ILM_EXPLAIN, INTERNAL_API_VERSION } from '../../common/constants'; import { fetchILMExplain } from '../lib'; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts index 34ce98d4f9378a..f1089b192ab059 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.test.ts @@ -12,7 +12,7 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getIndexMappingsRoute } from './get_index_mappings'; -import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ fetchMappings: jest.fn(), diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts index 6f1dfbf4ee8333..e593320933f7c6 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IRouter, Logger } from '@kbn/core/server'; +import type { IRouter, Logger } from '@kbn/core/server'; import { fetchMappings } from '../lib'; import { buildResponse } from '../lib/build_response'; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts index e000809797a01c..729c795e0665be 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.test.ts @@ -12,7 +12,8 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getIndexStatsRoute } from './get_index_stats'; -import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import type { MockedLogger } from '@kbn/logging-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ fetchStats: jest.fn(), diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts index 220c3e68141f07..cbaf7940a4b51d 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts @@ -5,9 +5,9 @@ * 2.0. */ import { i18n } from '@kbn/i18n'; -import { IRouter, Logger } from '@kbn/core/server'; +import type { IRouter, Logger } from '@kbn/core/server'; -import { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; +import type { IndicesStatsIndicesStats } from '@elastic/elasticsearch/lib/api/types'; import { fetchStats, fetchAvailableIndices } from '../lib'; import { buildResponse } from '../lib/build_response'; import { GET_INDEX_STATS, INTERNAL_API_VERSION } from '../../common/constants'; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts index fe18893acaec12..a283d24d1479e4 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.test.ts @@ -12,7 +12,8 @@ import { serverMock } from '../__mocks__/server'; import { requestMock } from '../__mocks__/request'; import { requestContextMock } from '../__mocks__/request_context'; import { getUnallowedFieldValuesRoute } from './get_unallowed_field_values'; -import { loggerMock, MockedLogger } from '@kbn/logging-mocks'; +import type { MockedLogger } from '@kbn/logging-mocks'; +import { loggerMock } from '@kbn/logging-mocks'; jest.mock('../lib', () => ({ getUnallowedFieldValues: jest.fn(), diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts index 8108396e8f39d3..76f0827caaad22 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_unallowed_field_values.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { IRouter, Logger } from '@kbn/core/server'; +import type { IRouter, Logger } from '@kbn/core/server'; import { getUnallowedFieldValues } from '../lib'; import { buildResponse } from '../lib/build_response'; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/index.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/index.ts index 6622471d463e3e..5ae8ebeae6103d 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/index.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/index.ts @@ -8,3 +8,4 @@ export { getIndexMappingsRoute } from './get_index_mappings'; export { getIndexStatsRoute } from './get_index_stats'; export { getUnallowedFieldValuesRoute } from './get_unallowed_field_values'; export { getILMExplainRoute } from './get_ilm_explain'; +export { resultsRoutes } from './results'; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts new file mode 100644 index 00000000000000..44f7a97abf0d08 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { RESULTS_ROUTE_PATH } from '../../../common/constants'; + +import { serverMock } from '../../__mocks__/server'; +import { requestMock } from '../../__mocks__/request'; +import { requestContextMock } from '../../__mocks__/request_context'; +import type { LatestAggResponseBucket } from './get_results'; +import { getResultsRoute, getQuery } from './get_results'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import { resultBody, resultDocument } from './results.mock'; +import type { + SearchResponse, + SecurityHasPrivilegesResponse, +} from '@elastic/elasticsearch/lib/api/types'; +import type { ResultDocument } from '../../schemas/result'; + +const searchResponse = { + aggregations: { + latest: { + buckets: [ + { + key: 'logs-*', + latest_doc: { hits: { hits: [{ _source: resultDocument }] } }, + }, + ], + }, + }, +} as unknown as SearchResponse< + ResultDocument, + Record +>; + +// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 +describe.skip('getResultsRoute route', () => { + describe('querying', () => { + let server: ReturnType; + let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + + const req = requestMock.create({ + method: 'get', + path: RESULTS_ROUTE_PATH, + query: { patterns: 'logs-*,alerts-*' }, + }); + + beforeEach(() => { + jest.clearAllMocks(); + + server = serverMock.create(); + logger = loggerMock.create(); + + ({ context } = requestContextMock.createTools()); + + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({ + index: { 'logs-*': { all: true }, 'alerts-*': { all: true } }, + } as unknown as SecurityHasPrivilegesResponse); + + getResultsRoute(server.router, logger); + }); + + it('gets result', async () => { + const mockSearch = context.core.elasticsearch.client.asInternalUser.search; + mockSearch.mockResolvedValueOnce(searchResponse); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(mockSearch).toHaveBeenCalled(); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([{ '@timestamp': expect.any(Number), ...resultBody }]); + }); + + it('handles results data stream error', async () => { + const errorMessage = 'Installation Error!'; + context.dataQualityDashboard.getResultsIndexName.mockRejectedValueOnce( + new Error(errorMessage) + ); + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(503); + expect(response.body).toEqual({ + message: expect.stringContaining(errorMessage), + status_code: 503, + }); + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining(errorMessage)); + }); + + it('handles error', async () => { + const errorMessage = 'Error!'; + const mockSearch = context.core.elasticsearch.client.asInternalUser.search; + mockSearch.mockRejectedValueOnce({ message: errorMessage }); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ message: errorMessage, status_code: 500 }); + }); + }); + + describe('request pattern authorization', () => { + let server: ReturnType; + let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + + const req = requestMock.create({ + method: 'get', + path: RESULTS_ROUTE_PATH, + query: { patterns: 'logs-*,alerts-*' }, + }); + + beforeEach(() => { + jest.clearAllMocks(); + + server = serverMock.create(); + logger = loggerMock.create(); + + ({ context } = requestContextMock.createTools()); + + context.core.elasticsearch.client.asInternalUser.search.mockResolvedValue(searchResponse); + + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({ + index: { 'logs-*': { all: true }, 'alerts-*': { all: true } }, + } as unknown as SecurityHasPrivilegesResponse); + + getResultsRoute(server.router, logger); + }); + + it('should authorize pattern', async () => { + const mockHasPrivileges = + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; + mockHasPrivileges.mockResolvedValueOnce({ + index: { 'logs-*': { all: true }, 'alerts-*': { all: true } }, + } as unknown as SecurityHasPrivilegesResponse); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(mockHasPrivileges).toHaveBeenCalledWith({ + index: [ + { names: ['logs-*', 'alerts-*'], privileges: ['all', 'read', 'view_index_metadata'] }, + ], + }); + expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalled(); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([{ '@timestamp': expect.any(Number), ...resultBody }]); + }); + + it('should search authorized patterns only', async () => { + const mockHasPrivileges = + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; + mockHasPrivileges.mockResolvedValueOnce({ + index: { 'logs-*': { all: false }, 'alerts-*': { all: true } }, + } as unknown as SecurityHasPrivilegesResponse); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({ + index: expect.any(String), + ...getQuery(['alerts-*']), + }); + + expect(response.status).toEqual(200); + }); + + it('should not search unauthorized patterns', async () => { + const mockHasPrivileges = + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; + mockHasPrivileges.mockResolvedValueOnce({ + index: { 'logs-*': { all: false }, 'alerts-*': { all: false } }, + } as unknown as SecurityHasPrivilegesResponse); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(context.core.elasticsearch.client.asInternalUser.search).not.toHaveBeenCalled(); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([]); + }); + + it('handles pattern authorization error', async () => { + const errorMessage = 'Error!'; + const mockHasPrivileges = + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; + mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage }); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ message: errorMessage, status_code: 500 }); + }); + }); + + describe('request validation', () => { + let server: ReturnType; + let logger: MockedLogger; + beforeEach(() => { + server = serverMock.create(); + logger = loggerMock.create(); + getResultsRoute(server.router, logger); + }); + + test('disallows invalid query param', () => { + const req = requestMock.create({ + method: 'get', + path: RESULTS_ROUTE_PATH, + query: {}, + }); + const result = server.validate(req); + + expect(result.badRequest).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts new file mode 100644 index 00000000000000..56729c7a40ab74 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter, Logger } from '@kbn/core/server'; + +import { RESULTS_ROUTE_PATH, INTERNAL_API_VERSION } from '../../../common/constants'; +import { buildResponse } from '../../lib/build_response'; +import { buildRouteValidation } from '../../schemas/common'; +import { GetResultQuery } from '../../schemas/result'; +import type { Result, ResultDocument } from '../../schemas/result'; +import { API_DEFAULT_ERROR_MESSAGE } from '../../translations'; +import type { DataQualityDashboardRequestHandlerContext } from '../../types'; +import { createResultFromDocument } from './parser'; +import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations'; + +export const getQuery = (patterns: string[]) => ({ + size: 0, + query: { + bool: { filter: [{ terms: { 'rollup.pattern': patterns } }] }, + }, + aggs: { + latest: { + terms: { field: 'rollup.pattern', size: 10000 }, // big enough to get all patterns, but under `index.max_terms_count` (default 65536) + aggs: { latest_doc: { top_hits: { size: 1, sort: [{ '@timestamp': { order: 'desc' } }] } } }, + }, + }, +}); + +export interface LatestAggResponseBucket { + key: string; + latest_doc: { hits: { hits: Array<{ _source: ResultDocument }> } }; +} + +export const getResultsRoute = ( + router: IRouter, + logger: Logger +) => { + router.versioned + .get({ + path: RESULTS_ROUTE_PATH, + access: 'internal', + options: { tags: ['access:securitySolution'] }, + }) + .addVersion( + { + version: INTERNAL_API_VERSION, + validate: { request: { query: buildRouteValidation(GetResultQuery) } }, + }, + async (context, request, response) => { + // TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 + return response.ok({ body: [] }); + + // eslint-disable-next-line no-unreachable + const services = await context.resolve(['core', 'dataQualityDashboard']); + const resp = buildResponse(response); + + let index: string; + try { + index = await services.dataQualityDashboard.getResultsIndexName(); + } catch (err) { + logger.error(`[GET results] Error retrieving results index name: ${err.message}`); + return resp.error({ + body: `${API_RESULTS_INDEX_NOT_AVAILABLE}: ${err.message}`, + statusCode: 503, + }); + } + + try { + // Confirm user has authorization for the requested patterns + const { patterns } = request.query; + const userEsClient = services.core.elasticsearch.client.asCurrentUser; + const privileges = await userEsClient.security.hasPrivileges({ + index: [ + { names: patterns.split(','), privileges: ['all', 'read', 'view_index_metadata'] }, + ], + }); + const authorizedPatterns = Object.keys(privileges.index).filter((pattern) => + Object.values(privileges.index[pattern]).some((v) => v === true) + ); + if (authorizedPatterns.length === 0) { + return response.ok({ body: [] }); + } + + // Get the latest result of each pattern + const query = { index, ...getQuery(authorizedPatterns) }; + const internalEsClient = services.core.elasticsearch.client.asInternalUser; + + const { aggregations } = await internalEsClient.search< + ResultDocument, + Record + >(query); + + const results: Result[] = + aggregations?.latest?.buckets.map((bucket) => + createResultFromDocument(bucket.latest_doc.hits.hits[0]._source) + ) ?? []; + + return response.ok({ body: results }); + } catch (err) { + logger.error(JSON.stringify(err)); + + return resp.error({ + body: err.message ?? API_DEFAULT_ERROR_MESSAGE, + statusCode: err.statusCode ?? 500, + }); + } + } + ); +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/index.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/index.ts new file mode 100644 index 00000000000000..25d4a913a19461 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter, Logger } from '@kbn/core/server'; + +import { postResultsRoute } from './post_results'; +import { getResultsRoute } from './get_results'; +import type { DataQualityDashboardRequestHandlerContext } from '../../types'; + +export const resultsRoutes = ( + router: IRouter, + logger: Logger +) => { + postResultsRoute(router, logger); + getResultsRoute(router, logger); +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.test.ts new file mode 100644 index 00000000000000..56800801ffc8f6 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.test.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createDocumentFromResult, createResultFromDocument } from './parser'; +import { resultBody, resultDocument } from './results.mock'; + +describe('createDocumentFromResult', () => { + it('should create document from result', () => { + const document = createDocumentFromResult(resultBody); + expect(document).toEqual({ ...resultDocument, '@timestamp': expect.any(Number) }); + }); +}); + +describe('createResultFromDocument', () => { + it('should create document from result', () => { + const result = createResultFromDocument(resultDocument); + expect(result).toEqual({ ...resultBody, '@timestamp': expect.any(Number) }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.ts new file mode 100644 index 00000000000000..198d5522839e45 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import type { Result, ResultDocument, IndexArray, IndexObject } from '../../schemas/result'; + +export const createDocumentFromResult = (result: Result): ResultDocument => { + const { rollup } = result; + const document: ResultDocument = { + ...result, + '@timestamp': Date.now(), + rollup: { + ...rollup, + ilmExplain: indexObjectToIndexArray(rollup.ilmExplain), + stats: indexObjectToIndexArray(rollup.stats), + results: indexObjectToIndexArray(rollup.results), + }, + }; + + return document; +}; + +export const createResultFromDocument = (document: ResultDocument): Result => { + const { rollup } = document; + const result = { + ...document, + rollup: { + ...rollup, + ilmExplain: indexArrayToIndexObject(rollup.ilmExplain), + stats: indexArrayToIndexObject(rollup.stats), + results: indexArrayToIndexObject(rollup.results), + }, + }; + + return result; +}; + +// ES parses object keys containing `.` as nested dot-separated field names (e.g. `event.name`). +// we need to convert documents containing objects with "indexName" keys (e.g. `.index-name-checked`) +// to object arrays so they can be stored correctly, we keep the key in the `_indexName` field. +const indexObjectToIndexArray = (obj: IndexObject): IndexArray => + Object.entries(obj).map(([key, value]) => ({ ...value, _indexName: key })); + +// convert index arrays back to objects with indexName as key +const indexArrayToIndexObject = (arr: IndexArray): IndexObject => + Object.fromEntries(arr.map(({ _indexName, ...value }) => [_indexName, value])); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts new file mode 100644 index 00000000000000..98eb67ecbaaa88 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { RESULTS_ROUTE_PATH } from '../../../common/constants'; + +import { serverMock } from '../../__mocks__/server'; +import { requestMock } from '../../__mocks__/request'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { postResultsRoute } from './post_results'; +import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; +import type { + SecurityHasPrivilegesResponse, + WriteResponseBase, +} from '@elastic/elasticsearch/lib/api/types'; +import { resultBody, resultDocument } from './results.mock'; + +// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 +describe.skip('postResultsRoute route', () => { + describe('indexation', () => { + let server: ReturnType; + let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + + const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, body: resultBody }); + + beforeEach(() => { + jest.clearAllMocks(); + + server = serverMock.create(); + logger = loggerMock.create(); + + ({ context } = requestContextMock.createTools()); + + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({ + has_all_requested: true, + } as unknown as SecurityHasPrivilegesResponse); + + postResultsRoute(server.router, logger); + }); + + it('indexes result', async () => { + const mockIndex = context.core.elasticsearch.client.asInternalUser.index; + mockIndex.mockResolvedValueOnce({ result: 'created' } as WriteResponseBase); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(mockIndex).toHaveBeenCalledWith({ + body: { ...resultDocument, '@timestamp': expect.any(Number) }, + index: await context.dataQualityDashboard.getResultsIndexName(), + }); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ result: 'created' }); + }); + + it('handles results data stream error', async () => { + const errorMessage = 'Installation Error!'; + context.dataQualityDashboard.getResultsIndexName.mockRejectedValueOnce( + new Error(errorMessage) + ); + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(503); + expect(response.body).toEqual({ + message: expect.stringContaining(errorMessage), + status_code: 503, + }); + expect(logger.error).toHaveBeenCalledWith(expect.stringContaining(errorMessage)); + }); + + it('handles index error', async () => { + const errorMessage = 'Error!'; + const mockIndex = context.core.elasticsearch.client.asInternalUser.index; + mockIndex.mockRejectedValueOnce({ message: errorMessage }); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ message: errorMessage, status_code: 500 }); + }); + }); + + describe('request pattern authorization', () => { + let server: ReturnType; + let { context } = requestContextMock.createTools(); + let logger: MockedLogger; + + const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, body: resultBody }); + + beforeEach(() => { + jest.clearAllMocks(); + + server = serverMock.create(); + logger = loggerMock.create(); + + ({ context } = requestContextMock.createTools()); + + context.core.elasticsearch.client.asInternalUser.index.mockResolvedValueOnce({ + result: 'created', + } as WriteResponseBase); + + postResultsRoute(server.router, logger); + }); + + it('should authorize pattern', async () => { + const mockHasPrivileges = + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; + mockHasPrivileges.mockResolvedValueOnce({ + has_all_requested: true, + } as unknown as SecurityHasPrivilegesResponse); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(mockHasPrivileges).toHaveBeenCalledWith({ + index: [ + { + names: [resultBody.rollup.pattern], + privileges: ['all', 'read', 'view_index_metadata'], + }, + ], + }); + expect(context.core.elasticsearch.client.asInternalUser.index).toHaveBeenCalled(); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ result: 'created' }); + }); + + it('should not index unauthorized pattern', async () => { + const mockHasPrivileges = + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; + mockHasPrivileges.mockResolvedValueOnce({ + has_all_requested: false, + } as unknown as SecurityHasPrivilegesResponse); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(mockHasPrivileges).toHaveBeenCalledWith({ + index: [ + { + names: [resultBody.rollup.pattern], + privileges: ['all', 'read', 'view_index_metadata'], + }, + ], + }); + expect(context.core.elasticsearch.client.asInternalUser.index).not.toHaveBeenCalled(); + + expect(response.status).toEqual(200); + expect(response.body).toEqual({ result: 'noop' }); + }); + + it('handles pattern authorization error', async () => { + const errorMessage = 'Error!'; + const mockHasPrivileges = + context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; + mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage }); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ message: errorMessage, status_code: 500 }); + }); + }); + + describe('request validation', () => { + let server: ReturnType; + let logger: MockedLogger; + beforeEach(() => { + server = serverMock.create(); + logger = loggerMock.create(); + postResultsRoute(server.router, logger); + }); + + test('disallows invalid pattern', () => { + const req = requestMock.create({ + method: 'post', + path: RESULTS_ROUTE_PATH, + body: { rollup: resultBody.rollup }, + }); + const result = server.validate(req); + + expect(result.badRequest).toHaveBeenCalled(); + }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.ts new file mode 100644 index 00000000000000..1162d23f1dfad7 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IRouter, Logger } from '@kbn/core/server'; + +import { RESULTS_ROUTE_PATH, INTERNAL_API_VERSION } from '../../../common/constants'; +import { buildResponse } from '../../lib/build_response'; +import { buildRouteValidation } from '../../schemas/common'; +import { PostResultBody } from '../../schemas/result'; +import { API_DEFAULT_ERROR_MESSAGE } from '../../translations'; +import type { DataQualityDashboardRequestHandlerContext } from '../../types'; +import { createDocumentFromResult } from './parser'; +import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations'; + +export const postResultsRoute = ( + router: IRouter, + logger: Logger +) => { + router.versioned + .post({ + path: RESULTS_ROUTE_PATH, + access: 'internal', + options: { tags: ['access:securitySolution'] }, + }) + .addVersion( + { + version: INTERNAL_API_VERSION, + validate: { request: { body: buildRouteValidation(PostResultBody) } }, + }, + async (context, request, response) => { + // TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 + return response.ok({ body: { result: 'noop' } }); + + // eslint-disable-next-line no-unreachable + const services = await context.resolve(['core', 'dataQualityDashboard']); + const resp = buildResponse(response); + + let index: string; + try { + index = await services.dataQualityDashboard.getResultsIndexName(); + } catch (err) { + logger.error(`[POST result] Error retrieving results index name: ${err.message}`); + return resp.error({ + body: `${API_RESULTS_INDEX_NOT_AVAILABLE}: ${err.message}`, + statusCode: 503, + }); + } + + try { + // Confirm user has authorization for the pattern payload + const { pattern } = request.body.rollup; + const userEsClient = services.core.elasticsearch.client.asCurrentUser; + const privileges = await userEsClient.security.hasPrivileges({ + index: [{ names: [pattern], privileges: ['all', 'read', 'view_index_metadata'] }], + }); + if (!privileges.has_all_requested) { + return response.ok({ body: { result: 'noop' } }); + } + + // Index the result + const document = createDocumentFromResult(request.body); + const esClient = services.core.elasticsearch.client.asInternalUser; + const outcome = await esClient.index({ index, body: document }); + + return response.ok({ body: { result: outcome.result } }); + } catch (err) { + logger.error(JSON.stringify(err)); + + return resp.error({ + body: err.message ?? API_DEFAULT_ERROR_MESSAGE, + statusCode: err.statusCode ?? 500, + }); + } + } + ); +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts new file mode 100644 index 00000000000000..1d0b15a4c24c01 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ResultDocument } from '../../schemas/result'; + +export const resultDocument: ResultDocument = { + '@timestamp': 1622767273955, + meta: { + batchId: 'aae36cd8-3825-4ad1-baa4-79bdf4617f8a', + ecsVersion: '8.6.1', + errorCount: 0, + ilmPhase: 'hot', + indexId: 'aO29KOwtQ3Snf-Pit5Wf4w', + indexName: '.internal.alerts-security.alerts-default-000001', + isCheckAll: true, + numberOfDocuments: 20, + numberOfFields: 1726, + numberOfIncompatibleFields: 2, + numberOfEcsFields: 1440, + numberOfCustomFields: 284, + numberOfIndices: 1, + numberOfIndicesChecked: 1, + numberOfSameFamily: 0, + sameFamilyFields: [], + sizeInBytes: 506471, + timeConsumedMs: 85, + unallowedMappingFields: [], + unallowedValueFields: ['event.category', 'event.outcome'], + }, + rollup: { + docsCount: 20, + error: null, + ilmExplain: [ + { + _indexName: '.internal.alerts-security.alerts-default-000001', + index: '.internal.alerts-security.alerts-default-000001', + managed: true, + policy: '.alerts-ilm-policy', + index_creation_date_millis: 1700757268526, + time_since_index_creation: '20.99d', + lifecycle_date_millis: 1700757268526, + age: '20.99d', + phase: 'hot', + phase_time_millis: 1700757270294, + action: 'rollover', + action_time_millis: 1700757273955, + step: 'check-rollover-ready', + step_time_millis: 1700757273955, + phase_execution: { + policy: '.alerts-ilm-policy', + phase_definition: { + min_age: '0ms', + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + version: 1, + modified_date_in_millis: 1700757266723, + }, + }, + ], + ilmExplainPhaseCounts: { + hot: 1, + warm: 0, + cold: 0, + frozen: 0, + unmanaged: 0, + }, + indices: 1, + pattern: '.alerts-security.alerts-default', + results: [ + { + _indexName: '.internal.alerts-security.alerts-default-000001', + docsCount: 20, + error: null, + ilmPhase: 'hot', + incompatible: 2, + indexName: '.internal.alerts-security.alerts-default-000001', + markdownComments: [ + '### .internal.alerts-security.alerts-default-000001\n', + '| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .internal.alerts-security.alerts-default-000001 | 20 (100,0 %) | 2 | `hot` | 494.6KB |\n\n', + '### **Incompatible fields** `2` **Same family** `0` **Custom fields** `284` **ECS compliant fields** `1440` **All fields** `1726`\n', + "#### 2 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n", + '\n\n#### Incompatible field values - .internal.alerts-security.alerts-default-000001\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `behavior` (1) |\n| event.outcome | `failure`, `success`, `unknown` | `` (12) |\n\n', + ], + pattern: '.alerts-security.alerts-default', + sameFamily: 0, + }, + ], + sizeInBytes: 506471, + stats: [ + { + _indexName: '.internal.alerts-security.alerts-default-000001', + uuid: 'aO29KOwtQ3Snf-Pit5Wf4w', + health: 'green', + status: 'open', + }, + ], + }, +}; + +export const resultBody = { + meta: { + batchId: 'aae36cd8-3825-4ad1-baa4-79bdf4617f8a', + ecsVersion: '8.6.1', + errorCount: 0, + ilmPhase: 'hot', + indexId: 'aO29KOwtQ3Snf-Pit5Wf4w', + indexName: '.internal.alerts-security.alerts-default-000001', + isCheckAll: true, + numberOfDocuments: 20, + numberOfFields: 1726, + numberOfIncompatibleFields: 2, + numberOfEcsFields: 1440, + numberOfCustomFields: 284, + numberOfIndices: 1, + numberOfIndicesChecked: 1, + numberOfSameFamily: 0, + sameFamilyFields: [], + sizeInBytes: 506471, + timeConsumedMs: 85, + unallowedMappingFields: [], + unallowedValueFields: ['event.category', 'event.outcome'], + }, + rollup: { + docsCount: 20, + error: null, + ilmExplain: { + '.internal.alerts-security.alerts-default-000001': { + index: '.internal.alerts-security.alerts-default-000001', + managed: true, + policy: '.alerts-ilm-policy', + index_creation_date_millis: 1700757268526, + time_since_index_creation: '20.99d', + lifecycle_date_millis: 1700757268526, + age: '20.99d', + phase: 'hot', + phase_time_millis: 1700757270294, + action: 'rollover', + action_time_millis: 1700757273955, + step: 'check-rollover-ready', + step_time_millis: 1700757273955, + phase_execution: { + policy: '.alerts-ilm-policy', + phase_definition: { + min_age: '0ms', + actions: { + rollover: { + max_age: '30d', + max_primary_shard_size: '50gb', + }, + }, + }, + version: 1, + modified_date_in_millis: 1700757266723, + }, + }, + }, + ilmExplainPhaseCounts: { + hot: 1, + warm: 0, + cold: 0, + frozen: 0, + unmanaged: 0, + }, + indices: 1, + pattern: '.alerts-security.alerts-default', + results: { + '.internal.alerts-security.alerts-default-000001': { + docsCount: 20, + error: null, + ilmPhase: 'hot', + incompatible: 2, + indexName: '.internal.alerts-security.alerts-default-000001', + markdownComments: [ + '### .internal.alerts-security.alerts-default-000001\n', + '| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .internal.alerts-security.alerts-default-000001 | 20 (100,0 %) | 2 | `hot` | 494.6KB |\n\n', + '### **Incompatible fields** `2` **Same family** `0` **Custom fields** `284` **ECS compliant fields** `1440` **All fields** `1726`\n', + "#### 2 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n", + '\n\n#### Incompatible field values - .internal.alerts-security.alerts-default-000001\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `behavior` (1) |\n| event.outcome | `failure`, `success`, `unknown` | `` (12) |\n\n', + ], + pattern: '.alerts-security.alerts-default', + sameFamily: 0, + }, + }, + sizeInBytes: 506471, + stats: { + '.internal.alerts-security.alerts-default-000001': { + uuid: 'aO29KOwtQ3Snf-Pit5Wf4w', + health: 'green', + status: 'open', + }, + }, + }, +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/translations.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/translations.ts new file mode 100644 index 00000000000000..924479e9da0ad9 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/translations.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; + +export const API_RESULTS_INDEX_NOT_AVAILABLE = i18n.translate( + 'xpack.ecsDataQualityDashboard.api.results.indexNotAvailable', + { + defaultMessage: 'Data Quality Dashboard result persistence not available', + } +); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts index 57dc45d4071f7c..00e97a9326c5e6 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/common.ts @@ -7,7 +7,7 @@ import { fold } from 'fp-ts/lib/Either'; import { pipe } from 'fp-ts/lib/pipeable'; -import * as rt from 'io-ts'; +import type * as rt from 'io-ts'; import { exactCheck, formatErrors } from '@kbn/securitysolution-io-ts-utils'; import type { RouteValidationFunction, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts new file mode 100644 index 00000000000000..09851c9b8dc86c --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import * as t from 'io-ts'; + +export const ResultMeta = t.type({ + batchId: t.string, + ecsVersion: t.string, + errorCount: t.number, + ilmPhase: t.string, + indexId: t.string, + indexName: t.string, + isCheckAll: t.boolean, + numberOfDocuments: t.number, + numberOfFields: t.number, + numberOfIncompatibleFields: t.number, + numberOfEcsFields: t.number, + numberOfCustomFields: t.number, + numberOfIndices: t.number, + numberOfIndicesChecked: t.number, + numberOfSameFamily: t.number, + sameFamilyFields: t.array(t.string), + sizeInBytes: t.number, + timeConsumedMs: t.number, + unallowedMappingFields: t.array(t.string), + unallowedValueFields: t.array(t.string), +}); +export type ResultMeta = t.TypeOf; + +export const ResultRollup = t.type({ + docsCount: t.number, + error: t.union([t.string, t.null]), + indices: t.number, + pattern: t.string, + sizeInBytes: t.number, + ilmExplainPhaseCounts: t.record(t.string, t.number), + ilmExplain: t.record(t.string, t.UnknownRecord), + stats: t.record(t.string, t.UnknownRecord), + results: t.record(t.string, t.UnknownRecord), +}); +export type ResultRollup = t.TypeOf; + +export const Result = t.type({ + meta: ResultMeta, + rollup: ResultRollup, +}); +export type Result = t.TypeOf; + +export type IndexArray = Array<{ _indexName: string } & Record>; +export type IndexObject = Record>; + +export type ResultDocument = Omit & { + '@timestamp': number; + rollup: Omit & { + stats: IndexArray; + results: IndexArray; + ilmExplain: IndexArray; + }; +}; + +// Routes validation schemas + +export const GetResultQuery = t.type({ patterns: t.string }); +export type GetResultQuery = t.TypeOf; + +export const PostResultBody = Result; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/types.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/types.ts index 0e7c1c68af6539..ed875cd83f8c45 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/types.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/types.ts @@ -5,8 +5,22 @@ * 2.0. */ +import type { CustomRequestHandlerContext } from '@kbn/core-http-request-handler-context-server'; +import type { SpacesPluginSetup } from '@kbn/spaces-plugin/server'; + /** The plugin setup interface */ export interface EcsDataQualityDashboardPluginSetup {} // eslint-disable-line @typescript-eslint/no-empty-interface /** The plugin start interface */ export interface EcsDataQualityDashboardPluginStart {} // eslint-disable-line @typescript-eslint/no-empty-interface + +export interface PluginSetupDependencies { + spaces?: SpacesPluginSetup; +} + +export type DataQualityDashboardRequestHandlerContext = CustomRequestHandlerContext<{ + dataQualityDashboard: { + spaceId: string; + getResultsIndexName: () => Promise; + }; +}>; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json index f742dc544a79bf..04a7d2bf092f55 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json +++ b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json @@ -21,6 +21,9 @@ "@kbn/i18n", "@kbn/core-http-router-server-mocks", "@kbn/logging-mocks", + "@kbn/data-stream-adapter", + "@kbn/spaces-plugin", + "@kbn/core-elasticsearch-server-mocks", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts index e39d36fb7fe0c2..6d3cb9167cb01d 100644 --- a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts +++ b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/index.ts @@ -77,6 +77,13 @@ export const dataQualityIndexCheckedEvent: DataQualityTelemetryIndexCheckedEvent optional: true, }, }, + numberOfFields: { + type: 'integer', + _meta: { + description: 'Total number of fields', + optional: true, + }, + }, numberOfIncompatibleFields: { type: 'integer', _meta: { @@ -84,6 +91,20 @@ export const dataQualityIndexCheckedEvent: DataQualityTelemetryIndexCheckedEvent optional: true, }, }, + numberOfEcsFields: { + type: 'integer', + _meta: { + description: 'Number of ecs compatible fields', + optional: true, + }, + }, + numberOfCustomFields: { + type: 'integer', + _meta: { + description: 'Number of custom fields', + optional: true, + }, + }, numberOfDocuments: { type: 'integer', _meta: { @@ -187,6 +208,13 @@ export const dataQualityCheckAllClickedEvent: DataQualityTelemetryCheckAllComple optional: true, }, }, + numberOfFields: { + type: 'integer', + _meta: { + description: 'Total number of fields', + optional: true, + }, + }, numberOfIncompatibleFields: { type: 'integer', _meta: { @@ -194,6 +222,20 @@ export const dataQualityCheckAllClickedEvent: DataQualityTelemetryCheckAllComple optional: true, }, }, + numberOfEcsFields: { + type: 'integer', + _meta: { + description: 'Number of ecs compatible fields', + optional: true, + }, + }, + numberOfCustomFields: { + type: 'integer', + _meta: { + description: 'Number of custom fields', + optional: true, + }, + }, numberOfDocuments: { type: 'integer', _meta: { diff --git a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts index 806ce3e741c314..6ae922d45b957f 100644 --- a/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts +++ b/x-pack/plugins/security_solution/public/common/lib/telemetry/events/data_quality/types.ts @@ -23,7 +23,10 @@ export interface ReportDataQualityCheckAllCompletedParams { ecsVersion?: string; isCheckAll?: boolean; numberOfDocuments?: number; + numberOfFields?: number; numberOfIncompatibleFields?: number; + numberOfEcsFields?: number; + numberOfCustomFields?: number; numberOfIndices?: number; numberOfIndicesChecked?: number; numberOfSameFamily?: number; diff --git a/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx b/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx index 0c318b38a4660a..1c2be8f8a798ed 100644 --- a/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx +++ b/x-pack/plugins/security_solution/public/overview/pages/data_quality.tsx @@ -136,12 +136,6 @@ const DataQualityComponent: React.FC = () => { const { baseTheme, theme } = useThemes(); const toasts = useToasts(); - const addSuccessToast = useCallback( - (toast: { title: string }) => { - toasts.addSuccess(toast); - }, - [toasts] - ); const [defaultBytesFormat] = useUiSetting$(DEFAULT_BYTES_FORMAT); const [defaultNumberFormat] = useUiSetting$(DEFAULT_NUMBER_FORMAT); const labelInputId = useGeneratedHtmlId({ prefix: 'labelInput' }); @@ -280,7 +274,6 @@ const DataQualityComponent: React.FC = () => { { setLastChecked={setLastChecked} startDate={startDate} theme={theme} + toasts={toasts} /> ) : ( diff --git a/yarn.lock b/yarn.lock index 05819606e4ec57..8cbbe1a512b0d5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4329,6 +4329,10 @@ version "0.0.0" uid "" +"@kbn/data-stream-adapter@link:packages/kbn-data-stream-adapter": + version "0.0.0" + uid "" + "@kbn/data-view-editor-plugin@link:src/plugins/data_view_editor": version "0.0.0" uid ""