diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 02837458e3046c..72f4dfddb686b3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -353,8 +353,8 @@ packages/kbn-docs-utils @elastic/kibana-operations packages/kbn-dom-drag-drop @elastic/kibana-visualizations @elastic/kibana-data-discovery packages/kbn-ebt-tools @elastic/kibana-core packages/kbn-ecs @elastic/kibana-core @elastic/security-threat-hunting-investigations -x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations -x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations +x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore +x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore packages/kbn-elastic-agent-utils @elastic/obs-ux-logs-team x-pack/packages/kbn-elastic-assistant @elastic/security-generative-ai x-pack/packages/kbn-elastic-assistant-common @elastic/security-generative-ai diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx index 3de5aba8bcc59d..6e1819d35537d6 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx @@ -44,7 +44,7 @@ import { useAddToNewCase } from '../../use_add_to_new_case'; import { useMappings } from '../../use_mappings'; import { useUnallowedValues } from '../../use_unallowed_values'; import { useDataQualityContext } from '../data_quality_context'; -import { getSizeInBytes, postResult } from '../../helpers'; +import { formatStorageResult, postStorageResult, getSizeInBytes } from '../../helpers'; const EMPTY_MARKDOWN_COMMENTS: string[] = []; @@ -249,6 +249,8 @@ const IndexPropertiesComponent: React.FC = ({ }) : EMPTY_MARKDOWN_COMMENTS; + const checkedAt = partitionedFieldMetadata ? Date.now() : undefined; + const updatedRollup = { ...patternRollup, results: { @@ -262,13 +264,14 @@ const IndexPropertiesComponent: React.FC = ({ markdownComments, pattern, sameFamily: indexSameFamily, + checkedAt, }, }, }; updatePatternRollup(updatedRollup); if (indexId && requestTime != null && requestTime > 0 && partitionedFieldMetadata) { - const checkMetadata = { + const report = { batchId: uuidv4(), ecsVersion: EcsVersion, errorCount: error ? 1 : 0, @@ -294,10 +297,13 @@ const IndexPropertiesComponent: React.FC = ({ partitionedFieldMetadata.incompatible ), }; - telemetryEvents.reportDataQualityIndexChecked?.(checkMetadata); + telemetryEvents.reportDataQualityIndexChecked?.(report); - const result = { meta: checkMetadata, rollup: updatedRollup }; - postResult({ result, httpFetch, toasts, abortController: new AbortController() }); + const result = updatedRollup.results[indexName]; + if (result) { + const storageResult = formatStorageResult({ result, report, partitionedFieldMetadata }); + postStorageResult({ storageResult, httpFetch, toasts }); + } } } } diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts index b2a7f6ef26a6e4..cdf62511137512 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts @@ -274,6 +274,7 @@ describe('helpers', () => { ], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: 1706526408000, }, }; const isILMAvailable = true; @@ -300,6 +301,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 733175040, + checkedAt: undefined, }, { docsCount: 1628343, @@ -309,6 +311,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 731583142, + checkedAt: undefined, }, { docsCount: 4, @@ -318,6 +321,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 28413, + checkedAt: 1706526408000, }, ]); }); @@ -344,6 +348,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 733175040, + checkedAt: undefined, }, { docsCount: 1628343, @@ -353,6 +358,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 731583142, + checkedAt: undefined, }, { docsCount: 4, @@ -362,6 +368,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 28413, + checkedAt: 1706526408000, }, ]); }); @@ -388,6 +395,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 28413, + checkedAt: 1706526408000, }, { docsCount: 1628343, @@ -397,6 +405,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 731583142, + checkedAt: undefined, }, { docsCount: 1630289, @@ -406,6 +415,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 733175040, + checkedAt: undefined, }, ]); }); @@ -432,6 +442,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 0, + checkedAt: undefined, }, { docsCount: 0, @@ -441,6 +452,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 0, + checkedAt: undefined, }, { docsCount: 0, @@ -450,6 +462,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 4, sizeInBytes: 0, + checkedAt: undefined, }, ]); }); @@ -701,6 +714,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 43357342, + checkedAt: 1706526408000, }, { docsCount: 48068, @@ -710,6 +724,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 32460397, + checkedAt: 1706526408000, }, { docsCount: 48064, @@ -719,6 +734,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 42782794, + checkedAt: 1706526408000, }, { docsCount: 47868, @@ -728,6 +744,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 31575964, + checkedAt: 1706526408000, }, { docsCount: 47827, @@ -737,6 +754,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 44130657, + checkedAt: 1706526408000, }, { docsCount: 47642, @@ -746,6 +764,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 42412521, + checkedAt: 1706526408000, }, { docsCount: 47545, @@ -755,6 +774,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 41423244, + checkedAt: 1706526408000, }, { docsCount: 47531, @@ -764,6 +784,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 32394133, + checkedAt: 1706526408000, }, { docsCount: 47530, @@ -773,6 +794,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 43015519, + checkedAt: 1706526408000, }, { docsCount: 47520, @@ -782,6 +804,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 42230604, + checkedAt: 1706526408000, }, { docsCount: 47496, @@ -791,6 +814,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 41710968, + checkedAt: 1706526408000, }, { docsCount: 47486, @@ -800,6 +824,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 42295944, + checkedAt: 1706526408000, }, { docsCount: 47486, @@ -809,6 +834,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 41761321, + checkedAt: 1706526408000, }, { docsCount: 47460, @@ -818,6 +844,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 30481198, + checkedAt: 1706526408000, }, { docsCount: 47439, @@ -827,6 +854,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 41554041, + checkedAt: 1706526408000, }, { docsCount: 47395, @@ -836,6 +864,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 42815907, + checkedAt: 1706526408000, }, { docsCount: 47394, @@ -845,6 +874,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 41157112, + checkedAt: 1706526408000, }, { docsCount: 47372, @@ -854,6 +884,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 31626792, + checkedAt: 1706526408000, }, { docsCount: 47369, @@ -863,6 +894,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 41828969, + checkedAt: 1706526408000, }, { docsCount: 47348, @@ -872,6 +904,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 40010773, + checkedAt: 1706526408000, }, { docsCount: 47339, @@ -881,6 +914,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 43480570, + checkedAt: 1706526408000, }, { docsCount: 47325, @@ -890,6 +924,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 41822475, + checkedAt: 1706526408000, }, { docsCount: 47294, @@ -899,6 +934,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 43018490, + checkedAt: 1706526408000, }, { docsCount: 24276, @@ -908,6 +944,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 23579440, + checkedAt: 1706526408000, }, { docsCount: 4, @@ -917,6 +954,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 28409, + checkedAt: 1706526408000, }, { docsCount: 0, @@ -926,6 +964,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 1118155, sizeInBytes: 247, + checkedAt: 1706526408000, }, ], pageSize: 10, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts index 82fd312a947e51..1bddcd83e7a141 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts @@ -172,6 +172,7 @@ export const getSummaryTableItems = ({ pattern, patternDocsCount, sizeInBytes: getSizeInBytes({ stats, indexName }), + checkedAt: results?.[indexName]?.checkedAt, })); return orderBy([sortByColumn], [sortByDirection], summaryTableItems); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx index 6ab874eeac5b61..cfcd23b86d2a13 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx @@ -143,6 +143,7 @@ describe('helpers', () => { pattern: 'auditbeat-*', patternDocsCount: 57410, sizeInBytes: 103344068, + checkedAt: Date.now(), }; const hasIncompatible: IndexSummaryTableItem = { @@ -188,6 +189,7 @@ describe('helpers', () => { }, { field: 'ilmPhase', name: 'ILM Phase', sortable: true, truncateText: false }, { field: 'sizeInBytes', name: 'Size', sortable: true, truncateText: false }, + { field: 'checkedAt', name: 'Last check', sortable: true, truncateText: false }, ]); }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx index f80678fff8cb20..55e26200f0e44a 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.tsx @@ -5,8 +5,9 @@ * 2.0. */ -import type { EuiBasicTableColumn } from '@elastic/eui'; import { + EuiBasicTableColumn, + EuiText, EuiBadge, EuiButtonIcon, EuiIcon, @@ -17,6 +18,7 @@ import { RIGHT_ALIGNMENT, } from '@elastic/eui'; import React from 'react'; +import moment from 'moment'; import styled from 'styled-components'; import { EMPTY_STAT, getIlmPhaseDescription, getIncompatibleStatColor } from '../../helpers'; @@ -41,6 +43,7 @@ export interface IndexSummaryTableItem { pattern: string; patternDocsCount: number; sizeInBytes: number; + checkedAt: number | undefined; } export const getResultToolTip = (incompatible: number | undefined): string => { @@ -237,6 +240,17 @@ export const getSummaryTableColumns = ({ sortable: true, truncateText: false, }, + { + field: 'checkedAt', + name: i18n.LAST_CHECK, + render: (_, { checkedAt }) => ( + + {checkedAt && moment(checkedAt).isValid() ? moment(checkedAt).fromNow() : EMPTY_STAT} + + ), + sortable: true, + truncateText: false, + }, ]; export const getShowPagination = ({ diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/translations.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/translations.ts index 0101708db9f9d4..8d8e4adb9944f2 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/translations.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/translations.ts @@ -111,6 +111,13 @@ export const SIZE = i18n.translate( } ); +export const LAST_CHECK = i18n.translate( + 'securitySolutionPackages.ecsDataQualityDashboard.summaryTable.lastCheckColumn', + { + defaultMessage: 'Last check', + } +); + export const THIS_INDEX_HAS_NOT_BEEN_CHECKED = i18n.translate( 'securitySolutionPackages.ecsDataQualityDashboard.summaryTable.thisIndexHasNotBeenCheckedTooltip', { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts index a20c8144c57738..830d463f5fe57b 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts @@ -35,9 +35,9 @@ import { getTotalSizeInBytes, hasValidTimestampMapping, isMappingCompatible, - postResult, - getResults, - ResultData, + postStorageResult, + getStorageResults, + StorageResult, } from './helpers'; import { hostNameWithTextMapping, @@ -102,6 +102,7 @@ describe('helpers', () => { ], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: Date.now(), }; it('returns undefined when results is undefined', () => { @@ -1193,6 +1194,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'packetbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, '.ds-packetbeat-8.6.1-2023.02.04-000001': { docsCount: 1628343, @@ -1203,6 +1205,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'packetbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, }; @@ -1220,6 +1223,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, 'auditbeat-custom-index-1': { docsCount: 4, @@ -1230,6 +1234,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, 'auditbeat-custom-empty-index-1': { docsCount: 0, @@ -1240,6 +1245,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, }; @@ -1257,6 +1263,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, 'auditbeat-custom-index-1': { docsCount: 4, @@ -1267,6 +1274,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, 'auditbeat-custom-empty-index-1': { docsCount: 0, @@ -1277,6 +1285,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: Date.now(), }, }; @@ -1342,6 +1351,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: 'packetbeat-*', sameFamily: 0, + checkedAt: Date.now(), }; expect(getErrorSummary(resultWithError)).toEqual({ @@ -1495,7 +1505,7 @@ describe('helpers', () => { }); }); - describe('postResult', () => { + describe('postStorageResult', () => { const { fetch } = httpServiceMock.createStartContract(); const { toasts } = notificationServiceMock.createStartContract(); beforeEach(() => { @@ -1503,10 +1513,10 @@ describe('helpers', () => { }); test('it posts the result', async () => { - const result = { meta: {}, rollup: {} } as unknown as ResultData; - await postResult({ + const storageResult = { indexName: 'test' } as unknown as StorageResult; + await postStorageResult({ + storageResult, httpFetch: fetch, - result, abortController: new AbortController(), toasts, }); @@ -1515,17 +1525,17 @@ describe('helpers', () => { '/internal/ecs_data_quality_dashboard/results', expect.objectContaining({ method: 'POST', - body: JSON.stringify(result), + body: JSON.stringify(storageResult), }) ); }); test('it throws error', async () => { - const result = { meta: {}, rollup: {} } as unknown as ResultData; + const storageResult = { indexName: 'test' } as unknown as StorageResult; fetch.mockRejectedValueOnce('test-error'); - await postResult({ + await postStorageResult({ httpFetch: fetch, - result, + storageResult, abortController: new AbortController(), toasts, }); @@ -1533,7 +1543,7 @@ describe('helpers', () => { }); }); - describe('getResults', () => { + describe('getStorageResults', () => { const { fetch } = httpServiceMock.createStartContract(); const { toasts } = notificationServiceMock.createStartContract(); beforeEach(() => { @@ -1541,10 +1551,10 @@ describe('helpers', () => { }); test('it gets the results', async () => { - await getResults({ + await getStorageResults({ httpFetch: fetch, abortController: new AbortController(), - patterns: ['auditbeat-*', 'packetbeat-*'], + pattern: 'auditbeat-*', toasts, }); @@ -1552,7 +1562,7 @@ describe('helpers', () => { '/internal/ecs_data_quality_dashboard/results', expect.objectContaining({ method: 'GET', - query: { patterns: 'auditbeat-*,packetbeat-*' }, + query: { pattern: 'auditbeat-*' }, }) ); }); @@ -1560,10 +1570,10 @@ describe('helpers', () => { it('should catch error', async () => { fetch.mockRejectedValueOnce('test-error'); - const results = await getResults({ + const results = await getStorageResults({ httpFetch: fetch, abortController: new AbortController(), - patterns: ['auditbeat-*', 'packetbeat-*'], + pattern: 'auditbeat-*', toasts, }); diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts index a4d51233232e46..2107e7d3949daf 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts @@ -23,6 +23,7 @@ import type { EcsMetadata, EnrichedFieldMetadata, ErrorSummary, + IlmPhase, PartitionedFieldMetadata, PartitionedFieldMetadataStats, PatternRollup, @@ -449,51 +450,116 @@ export const getErrorSummaries = ( export const RESULTS_API_ROUTE = '/internal/ecs_data_quality_dashboard/results'; -export interface ResultData { - meta: DataQualityIndexCheckedParams; - rollup: PatternRollup; +export interface StorageResult { + batchId: string; + indexName: string; + isCheckAll: boolean; + checkedAt: number; + docsCount: number; + totalFieldCount: number; + ecsFieldCount: number; + customFieldCount: number; + incompatibleFieldCount: number; + sameFamilyFieldCount: number; + sameFamilyFields: string[]; + unallowedMappingFields: string[]; + unallowedValueFields: string[]; + sizeInBytes: number; + ilmPhase?: IlmPhase; + markdownComments: string[]; + ecsVersion: string; + indexId: string; + error: string | null; } -export async function postResult({ +export const formatStorageResult = ({ result, + report, + partitionedFieldMetadata, +}: { + result: DataQualityCheckResult; + report: DataQualityIndexCheckedParams; + partitionedFieldMetadata: PartitionedFieldMetadata; +}): StorageResult => ({ + batchId: report.batchId, + indexName: result.indexName, + isCheckAll: report.isCheckAll, + checkedAt: result.checkedAt ?? Date.now(), + docsCount: result.docsCount ?? 0, + totalFieldCount: partitionedFieldMetadata.all.length, + ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length, + customFieldCount: partitionedFieldMetadata.custom.length, + incompatibleFieldCount: partitionedFieldMetadata.incompatible.length, + sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length, + sameFamilyFields: report.sameFamilyFields ?? [], + unallowedMappingFields: report.unallowedMappingFields ?? [], + unallowedValueFields: report.unallowedValueFields ?? [], + sizeInBytes: report.sizeInBytes ?? 0, + ilmPhase: result.ilmPhase, + markdownComments: result.markdownComments, + ecsVersion: report.ecsVersion, + indexId: report.indexId, + error: result.error, +}); + +export const formatResultFromStorage = ({ + storageResult, + pattern, +}: { + storageResult: StorageResult; + pattern: string; +}): DataQualityCheckResult => ({ + docsCount: storageResult.docsCount, + error: storageResult.error, + ilmPhase: storageResult.ilmPhase, + incompatible: storageResult.incompatibleFieldCount, + indexName: storageResult.indexName, + markdownComments: storageResult.markdownComments, + sameFamily: storageResult.sameFamilyFieldCount, + checkedAt: storageResult.checkedAt, + pattern, +}); + +export async function postStorageResult({ + storageResult, httpFetch, toasts, - abortController, + abortController = new AbortController(), }: { - result: ResultData; + storageResult: StorageResult; httpFetch: HttpHandler; toasts: IToasts; - abortController: AbortController; + abortController?: AbortController; }): Promise { try { await httpFetch(RESULTS_API_ROUTE, { method: 'POST', signal: abortController.signal, version: INTERNAL_API_VERSION, - body: JSON.stringify(result), + body: JSON.stringify(storageResult), }); } catch (err) { toasts.addError(err, { title: i18n.POST_RESULT_ERROR_TITLE }); } } -export async function getResults({ - patterns, +export async function getStorageResults({ + pattern, httpFetch, toasts, abortController, }: { - patterns: string[]; + pattern: string; httpFetch: HttpHandler; toasts: IToasts; abortController: AbortController; -}): Promise { +}): Promise { try { - const results = await httpFetch(RESULTS_API_ROUTE, { + const results = await httpFetch(RESULTS_API_ROUTE, { method: 'GET', signal: abortController.signal, version: INTERNAL_API_VERSION, - query: { patterns: patterns.join(',') }, + query: { pattern }, }); return results; } catch (err) { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/data_quality_check_result/mock_index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/data_quality_check_result/mock_index.tsx index f179a82c232e99..7f61f91b87212b 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/data_quality_check_result/mock_index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/data_quality_check_result/mock_index.tsx @@ -23,6 +23,7 @@ export const mockDataQualityCheckResult: Record ], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: 1706526408000, }, 'auditbeat-7.9.3-2023.02.13-000001': { docsCount: 2438, @@ -39,5 +40,6 @@ export const mockDataQualityCheckResult: Record ], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: 1706526408000, }, }; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts index f121621ff1ac10..513d034f522636 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_alerts_pattern_rollup.ts @@ -83,6 +83,7 @@ export const alertIndexWithAllResults: PatternRollup = { markdownComments: ['foo', 'bar', 'baz'], pattern: '.alerts-security.alerts-default', sameFamily: 0, + checkedAt: 1706526408000, }, }, sizeInBytes: 29717961631, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts index 2f83f899dc0d21..15bb39a01714f2 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_auditbeat_pattern_rollup.ts @@ -155,6 +155,7 @@ export const auditbeatWithAllResults: PatternRollup = { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: 1706526408000, }, 'auditbeat-custom-index-1': { docsCount: 4, @@ -165,6 +166,7 @@ export const auditbeatWithAllResults: PatternRollup = { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: 1706526408000, }, 'auditbeat-custom-empty-index-1': { docsCount: 0, @@ -175,6 +177,7 @@ export const auditbeatWithAllResults: PatternRollup = { markdownComments: ['foo', 'bar', 'baz'], pattern: 'auditbeat-*', sameFamily: 0, + checkedAt: 1706526408000, }, }, sizeInBytes: 18820446, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts index 369803a44a3dd1..339f1993e292d0 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/mock/pattern_rollup/mock_packetbeat_pattern_rollup.ts @@ -132,6 +132,7 @@ export const packetbeatWithSomeErrors: PatternRollup = { markdownComments: ['foo', 'bar', 'baz'], pattern: 'packetbeat-*', sameFamily: undefined, + checkedAt: 1706526408000, }, '.ds-packetbeat-8.6.1-2023.02.04-000001': { docsCount: 1628343, @@ -142,6 +143,7 @@ export const packetbeatWithSomeErrors: PatternRollup = { markdownComments: ['foo', 'bar', 'baz'], pattern: 'packetbeat-*', sameFamily: 0, + checkedAt: 1706526408000, }, }, sizeInBytes: 1096520898, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts index 9f507992d15093..f462777e1fc0bd 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts @@ -107,6 +107,7 @@ export interface DataQualityCheckResult { markdownComments: string[]; sameFamily: number | undefined; pattern: string; + checkedAt: number | undefined; } export interface PatternRollup { @@ -186,8 +187,8 @@ export type DataQualityIndexCheckedParams = DataQualityCheckAllCompletedParams & export interface DataQualityCheckAllCompletedParams { batchId: string; - ecsVersion?: string; - isCheckAll?: boolean; + ecsVersion: string; + isCheckAll: boolean; numberOfDocuments?: number; numberOfIncompatibleFields?: number; numberOfIndices?: number; diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts index b37d0aecc25cfe..b948b9584f996d 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.test.ts @@ -61,6 +61,7 @@ describe('helpers', () => { markdownComments: ['foo', 'bar', 'baz'], pattern: '.alerts-security.alerts-default', sameFamily: 7, + checkedAt: 1706526408000, }; const alertIndexWithSameFamily: PatternRollup = { @@ -260,6 +261,7 @@ describe('helpers', () => { ], pattern: 'packetbeat-*', sameFamily: 0, + checkedAt: expect.any(Number), }, }, sizeInBytes: 1464758182, @@ -361,6 +363,7 @@ describe('helpers', () => { ], pattern: 'packetbeat-*', sameFamily: 0, + checkedAt: expect.any(Number), }, }, sizeInBytes: 1464758182, @@ -450,6 +453,7 @@ describe('helpers', () => { indexName: '.ds-packetbeat-8.6.1-2023.02.04-000001', markdownComments: [], pattern: 'packetbeat-*', + checkedAt: undefined, }, }, sizeInBytes: 1464758182, @@ -510,6 +514,7 @@ describe('helpers', () => { ], pattern: 'packetbeat-*', sameFamily: 0, + checkedAt: expect.any(Number), }, }, sizeInBytes: 1464758182, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts index cca8ac331aa885..07f51572b6ba23 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/helpers.ts @@ -140,6 +140,7 @@ export const updateResultOnCheckCompleted = ({ const incompatible = partitionedFieldMetadata?.incompatible.length; const sameFamily = partitionedFieldMetadata?.sameFamily.length; + const checkedAt = partitionedFieldMetadata ? Date.now() : undefined; return { ...patternRollups, @@ -156,6 +157,7 @@ export const updateResultOnCheckCompleted = ({ markdownComments, pattern, sameFamily, + checkedAt, }, }, }, diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx index 27810fedffde4f..907ffca5a45f68 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/use_results_rollup/index.tsx @@ -19,14 +19,16 @@ import { updateResultOnCheckCompleted, } from './helpers'; -import type { OnCheckCompleted, PatternRollup } from '../types'; +import type { DataQualityCheckResult, OnCheckCompleted, PatternRollup } from '../types'; import { getDocsCount, getIndexId, - getResults, + getStorageResults, getSizeInBytes, getTotalPatternSameFamily, - postResult, + postStorageResult, + formatStorageResult, + formatResultFromStorage, } from '../helpers'; import { getIlmPhase, getIndexIncompatible } from '../data_quality_panel/pattern/helpers'; import { useDataQualityContext } from '../data_quality_panel/data_quality_context'; @@ -60,9 +62,11 @@ interface UseResultsRollup { updatePatternRollup: (patternRollup: PatternRollup) => void; } -const useStoredPatternRollups = (patterns: string[]) => { +const useStoredPatternResults = (patterns: string[]) => { const { httpFetch, toasts } = useDataQualityContext(); - const [storedRollups, setStoredRollups] = useState>({}); + const [storedPatternResults, setStoredPatternResults] = useState< + Array<{ pattern: string; results: Record }> + >([]); useEffect(() => { if (isEmpty(patterns)) { @@ -71,20 +75,31 @@ const useStoredPatternRollups = (patterns: string[]) => { let ignore = false; const abortController = new AbortController(); - const fetchStoredRollups = async () => { - const results = await getResults({ httpFetch, abortController, patterns, toasts }); - if (results?.length && !ignore) { - setStoredRollups(Object.fromEntries(results.map(({ rollup }) => [rollup.pattern, rollup]))); + const fetchStoredPatternResults = async () => { + const requests = patterns.map((pattern) => + getStorageResults({ pattern, httpFetch, abortController, toasts }).then((results = []) => ({ + pattern, + results: Object.fromEntries( + results.map((storageResult) => [ + storageResult.indexName, + formatResultFromStorage({ storageResult, pattern }), + ]) + ), + })) + ); + const patternResults = await Promise.all(requests); + if (patternResults?.length && !ignore) { + setStoredPatternResults(patternResults); } }; - fetchStoredRollups(); + fetchStoredPatternResults(); return () => { ignore = true; }; }, [httpFetch, patterns, toasts]); - return storedRollups; + return storedPatternResults; }; export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRollup => { @@ -92,28 +107,36 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll const [patternIndexNames, setPatternIndexNames] = useState>({}); const [patternRollups, setPatternRollups] = useState>({}); - const storedPatternsRollups = useStoredPatternRollups(patterns); + const storedPatternsResults = useStoredPatternResults(patterns); useEffect(() => { - if (!isEmpty(storedPatternsRollups)) { - setPatternRollups((current) => ({ ...current, ...storedPatternsRollups })); + if (!isEmpty(storedPatternsResults)) { + setPatternRollups((current) => + storedPatternsResults.reduce( + (acc, { pattern, results }) => ({ + ...acc, + [pattern]: { + ...current[pattern], + pattern, + results, + }, + }), + current + ) + ); } - }, [storedPatternsRollups]); - - const updatePatternRollups = useCallback( - (updateRollups: (current: Record) => Record) => { - setPatternRollups((current) => updateRollups(current)); - }, - [] - ); + }, [storedPatternsResults]); const { telemetryEvents, isILMAvailable } = useDataQualityContext(); - const updatePatternRollup = useCallback( - (patternRollup: PatternRollup) => { - updatePatternRollups((current) => ({ ...current, [patternRollup.pattern]: patternRollup })); - }, - [updatePatternRollups] - ); + const updatePatternRollup = useCallback((patternRollup: PatternRollup) => { + setPatternRollups((current) => ({ + ...current, + [patternRollup.pattern]: { + ...patternRollup, + results: patternRollup.results ?? current[patternRollup.pattern]?.results, // prevent undefined results override existing + }, + })); + }, []); const totalDocsCount = useMemo(() => getTotalDocsCount(patternRollups), [patternRollups]); const totalIncompatible = useMemo(() => getTotalIncompatible(patternRollups), [patternRollups]); @@ -170,7 +193,7 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll requestTime > 0 && partitionedFieldMetadata ) { - const metadata = { + const report = { batchId, ecsVersion: EcsVersion, errorCount: error ? 1 : 0, @@ -199,10 +222,13 @@ export const useResultsRollup = ({ ilmPhases, patterns }: Props): UseResultsRoll partitionedFieldMetadata.incompatible ), }; - telemetryEvents.reportDataQualityIndexChecked?.(metadata); + telemetryEvents.reportDataQualityIndexChecked?.(report); - const result = { meta: metadata, rollup: updatedRollup }; - postResult({ result, httpFetch, toasts, abortController: new AbortController() }); + const result = results[indexName]; + if (result) { + const storageResult = formatStorageResult({ result, report, partitionedFieldMetadata }); + postStorageResult({ storageResult, httpFetch, toasts }); + } } if (isLastCheck) { diff --git a/x-pack/packages/security-solution/ecs_data_quality_dashboard/kibana.jsonc b/x-pack/packages/security-solution/ecs_data_quality_dashboard/kibana.jsonc index a001420fade88f..30cc9cf2498205 100644 --- a/x-pack/packages/security-solution/ecs_data_quality_dashboard/kibana.jsonc +++ b/x-pack/packages/security-solution/ecs_data_quality_dashboard/kibana.jsonc @@ -1,5 +1,5 @@ { "type": "shared-browser", "id": "@kbn/ecs-data-quality-dashboard", - "owner": "@elastic/security-threat-hunting-investigations" + "owner": "@elastic/security-threat-hunting-explore" } diff --git a/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc b/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc index 2650184783066f..5adbe3eeee830b 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc +++ b/x-pack/plugins/ecs_data_quality_dashboard/kibana.jsonc @@ -1,7 +1,7 @@ { "type": "plugin", "id": "@kbn/ecs-data-quality-dashboard-plugin", - "owner": "@elastic/security-threat-hunting-investigations", + "owner": "@elastic/security-threat-hunting-explore", "description": "APIs used to assess the quality of data in Elasticsearch indexes", "plugin": { "id": "ecsDataQualityDashboard", diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts index 59f8ade6cb8343..c0b929ec6deb8b 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/lib/data_stream/results_field_map.ts @@ -8,33 +8,23 @@ import type { FieldMap } from '@kbn/data-stream-adapter'; export const resultsFieldMap: FieldMap = { - 'meta.batchId': { type: 'keyword', required: true }, - 'meta.ecsVersion': { type: 'keyword', required: true }, - 'meta.errorCount': { type: 'long', required: true }, - 'meta.ilmPhase': { type: 'keyword', required: true }, - 'meta.indexId': { type: 'keyword', required: true }, - 'meta.indexName': { type: 'keyword', required: true }, - 'meta.isCheckAll': { type: 'boolean', required: true }, - 'meta.numberOfDocuments': { type: 'long', required: true }, - 'meta.numberOfFields': { type: 'long', required: true }, - 'meta.numberOfIncompatibleFields': { type: 'long', required: true }, - 'meta.numberOfEcsFields': { type: 'long', required: true }, - 'meta.numberOfCustomFields': { type: 'long', required: true }, - 'meta.numberOfIndices': { type: 'long', required: true }, - 'meta.numberOfIndicesChecked': { type: 'long', required: true }, - 'meta.numberOfSameFamily': { type: 'long', required: true }, - 'meta.sameFamilyFields': { type: 'keyword', required: true, array: true }, - 'meta.sizeInBytes': { type: 'long', required: true }, - 'meta.timeConsumedMs': { type: 'long', required: true }, - 'meta.unallowedMappingFields': { type: 'keyword', required: true, array: true }, - 'meta.unallowedValueFields': { type: 'keyword', required: true, array: true }, - 'rollup.docsCount': { type: 'long', required: true }, - 'rollup.error': { type: 'text', required: false }, - 'rollup.ilmExplainPhaseCounts': { type: 'object', required: false }, - 'rollup.indices': { type: 'long', required: true }, - 'rollup.pattern': { type: 'keyword', required: true }, - 'rollup.sizeInBytes': { type: 'long', required: true }, - 'rollup.ilmExplain': { type: 'object', required: true, array: true }, - 'rollup.stats': { type: 'object', required: true, array: true }, - 'rollup.results': { type: 'object', required: true, array: true }, + batchId: { type: 'keyword', required: true }, + indexName: { type: 'keyword', required: true }, + isCheckAll: { type: 'boolean', required: true }, + checkedAt: { type: 'date', required: true }, + docsCount: { type: 'long', required: true }, + totalFieldCount: { type: 'long', required: true }, + ecsFieldCount: { type: 'long', required: true }, + customFieldCount: { type: 'long', required: true }, + incompatibleFieldCount: { type: 'long', required: true }, + sameFamilyFieldCount: { type: 'long', required: true }, + sameFamilyFields: { type: 'keyword', required: true, array: true }, + unallowedMappingFields: { type: 'keyword', required: true, array: true }, + unallowedValueFields: { type: 'keyword', required: true, array: true }, + sizeInBytes: { type: 'long', required: true }, + ilmPhase: { type: 'keyword', required: true }, + markdownComments: { type: 'text', required: true, array: true }, + ecsVersion: { type: 'keyword', required: true }, + indexId: { type: 'keyword', required: true }, + error: { type: 'text', required: false }, }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts index 19c6f12479694a..cb6fe9da7c2768 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/plugin.ts @@ -48,14 +48,13 @@ export class EcsDataQualityDashboardPlugin public setup(core: CoreSetup, plugins: PluginSetupDependencies) { this.logger.debug('ecsDataQualityDashboard: Setup'); - // TODO: Uncomment https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 - // this.resultsDataStream.install({ - // esClient: core - // .getStartServices() - // .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), - // logger: this.logger, - // pluginStop$: this.pluginStop$, - // }); + this.resultsDataStream.install({ + esClient: core + .getStartServices() + .then(([{ elasticsearch }]) => elasticsearch.client.asInternalUser), + logger: this.logger, + pluginStop$: this.pluginStop$, + }); core.http.registerRouteHandlerContext< DataQualityDashboardRequestHandlerContext, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts index 73282d11e3d71e..31202adffed2c5 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_ilm_explain.ts @@ -19,6 +19,7 @@ export const getILMExplainRoute = (router: IRouter, logger: Logger) => { .get({ path: GET_ILM_EXPLAIN, access: 'internal', + options: { tags: ['access:securitySolution'] }, }) .addVersion( { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts index e593320933f7c6..f3c59ccf9f3e21 100755 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_mappings.ts @@ -19,6 +19,7 @@ export const getIndexMappingsRoute = (router: IRouter, logger: Logger) => { .get({ path: GET_INDEX_MAPPINGS, access: 'internal', + options: { tags: ['access:securitySolution'] }, }) .addVersion( { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts index cbaf7940a4b51d..69d49b86111016 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/get_index_stats.ts @@ -20,6 +20,7 @@ export const getIndexStatsRoute = (router: IRouter, logger: Logger) => { .get({ path: GET_INDEX_STATS, access: 'internal', + options: { tags: ['access:securitySolution'] }, }) .addVersion( { diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts index 44f7a97abf0d08..05a714a27275a7 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.test.ts @@ -12,19 +12,17 @@ import { requestContextMock } from '../../__mocks__/request_context'; import type { LatestAggResponseBucket } from './get_results'; import { getResultsRoute, getQuery } from './get_results'; import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; -import { resultBody, resultDocument } from './results.mock'; -import type { - SearchResponse, - SecurityHasPrivilegesResponse, -} from '@elastic/elasticsearch/lib/api/types'; +import { resultDocument } from './results.mock'; +import type { SearchResponse } from '@elastic/elasticsearch/lib/api/types'; import type { ResultDocument } from '../../schemas/result'; +import type { CheckIndicesPrivilegesParam } from './privileges'; const searchResponse = { aggregations: { latest: { buckets: [ { - key: 'logs-*', + key: resultDocument.indexName, latest_doc: { hits: { hits: [{ _source: resultDocument }] } }, }, ], @@ -35,8 +33,15 @@ const searchResponse = { Record >; -// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 -describe.skip('getResultsRoute route', () => { +const mockCheckIndicesPrivileges = jest.fn(({ indices }: CheckIndicesPrivilegesParam) => + Promise.resolve(Object.fromEntries(indices.map((index) => [index, true]))) +); +jest.mock('./privileges', () => ({ + checkIndicesPrivileges: (params: CheckIndicesPrivilegesParam) => + mockCheckIndicesPrivileges(params), +})); + +describe('getResultsRoute route', () => { describe('querying', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); @@ -45,7 +50,7 @@ describe.skip('getResultsRoute route', () => { const req = requestMock.create({ method: 'get', path: RESULTS_ROUTE_PATH, - query: { patterns: 'logs-*,alerts-*' }, + query: { pattern: 'logs-*' }, }); beforeEach(() => { @@ -56,9 +61,9 @@ describe.skip('getResultsRoute route', () => { ({ context } = requestContextMock.createTools()); - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({ - index: { 'logs-*': { all: true }, 'alerts-*': { all: true } }, - } as unknown as SecurityHasPrivilegesResponse); + context.core.elasticsearch.client.asInternalUser.indices.get.mockResolvedValue({ + [resultDocument.indexName]: {}, + }); getResultsRoute(server.router, logger); }); @@ -68,10 +73,13 @@ describe.skip('getResultsRoute route', () => { mockSearch.mockResolvedValueOnce(searchResponse); const response = await server.inject(req, requestContextMock.convertContext(context)); - expect(mockSearch).toHaveBeenCalled(); + expect(mockSearch).toHaveBeenCalledWith({ + index: expect.any(String), + ...getQuery([resultDocument.indexName]), + }); expect(response.status).toEqual(200); - expect(response.body).toEqual([{ '@timestamp': expect.any(Number), ...resultBody }]); + expect(response.body).toEqual([resultDocument]); }); it('handles results data stream error', async () => { @@ -99,7 +107,7 @@ describe.skip('getResultsRoute route', () => { }); }); - describe('request pattern authorization', () => { + describe('request indices authorization', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); let logger: MockedLogger; @@ -107,7 +115,7 @@ describe.skip('getResultsRoute route', () => { const req = requestMock.create({ method: 'get', path: RESULTS_ROUTE_PATH, - query: { patterns: 'logs-*,alerts-*' }, + query: { pattern: 'logs-*' }, }); beforeEach(() => { @@ -120,54 +128,69 @@ describe.skip('getResultsRoute route', () => { context.core.elasticsearch.client.asInternalUser.search.mockResolvedValue(searchResponse); - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({ - index: { 'logs-*': { all: true }, 'alerts-*': { all: true } }, - } as unknown as SecurityHasPrivilegesResponse); + context.core.elasticsearch.client.asInternalUser.indices.get.mockResolvedValue({ + [resultDocument.indexName]: {}, + }); getResultsRoute(server.router, logger); }); - it('should authorize pattern', async () => { - const mockHasPrivileges = - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; - mockHasPrivileges.mockResolvedValueOnce({ - index: { 'logs-*': { all: true }, 'alerts-*': { all: true } }, - } as unknown as SecurityHasPrivilegesResponse); + it('should authorize indices from pattern', async () => { + const mockGetIndices = context.core.elasticsearch.client.asInternalUser.indices.get; + mockGetIndices.mockResolvedValueOnce({ [resultDocument.indexName]: {} }); const response = await server.inject(req, requestContextMock.convertContext(context)); - expect(mockHasPrivileges).toHaveBeenCalledWith({ - index: [ - { names: ['logs-*', 'alerts-*'], privileges: ['all', 'read', 'view_index_metadata'] }, - ], - }); + expect(mockGetIndices).toHaveBeenCalledWith({ index: 'logs-*', features: 'aliases' }); + expect(mockCheckIndicesPrivileges).toHaveBeenCalledWith( + expect.objectContaining({ indices: [resultDocument.indexName] }) + ); expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalled(); expect(response.status).toEqual(200); - expect(response.body).toEqual([{ '@timestamp': expect.any(Number), ...resultBody }]); + expect(response.body).toEqual([resultDocument]); }); - it('should search authorized patterns only', async () => { - const mockHasPrivileges = - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; - mockHasPrivileges.mockResolvedValueOnce({ - index: { 'logs-*': { all: false }, 'alerts-*': { all: true } }, - } as unknown as SecurityHasPrivilegesResponse); + it('should authorize data streams from pattern', async () => { + const dataStreamName = 'test_data_stream_name'; + const resultIndexNameTwo = `${resultDocument.indexName}_2`; + const resultIndexNameThree = `${resultDocument.indexName}_3`; + const mockGetIndices = context.core.elasticsearch.client.asInternalUser.indices.get; + mockGetIndices.mockResolvedValueOnce({ + [resultDocument.indexName]: {}, + [resultIndexNameTwo]: { data_stream: dataStreamName }, + [resultIndexNameThree]: { data_stream: dataStreamName }, + }); const response = await server.inject(req, requestContextMock.convertContext(context)); + + expect(mockGetIndices).toHaveBeenCalledWith({ index: 'logs-*', features: 'aliases' }); + expect(mockCheckIndicesPrivileges).toHaveBeenCalledWith( + expect.objectContaining({ indices: [resultDocument.indexName, dataStreamName] }) + ); expect(context.core.elasticsearch.client.asInternalUser.search).toHaveBeenCalledWith({ index: expect.any(String), - ...getQuery(['alerts-*']), + ...getQuery([resultDocument.indexName, resultIndexNameTwo, resultIndexNameThree]), }); expect(response.status).toEqual(200); + expect(response.body).toEqual([resultDocument]); + }); + + it('should not search unknown indices', async () => { + const mockGetIndices = context.core.elasticsearch.client.asInternalUser.indices.get; + mockGetIndices.mockResolvedValueOnce({}); // empty object means no index is found + + const response = await server.inject(req, requestContextMock.convertContext(context)); + + expect(mockCheckIndicesPrivileges).not.toHaveBeenCalled(); + expect(context.core.elasticsearch.client.asInternalUser.search).not.toHaveBeenCalled(); + + expect(response.status).toEqual(200); + expect(response.body).toEqual([]); }); - it('should not search unauthorized patterns', async () => { - const mockHasPrivileges = - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; - mockHasPrivileges.mockResolvedValueOnce({ - index: { 'logs-*': { all: false }, 'alerts-*': { all: false } }, - } as unknown as SecurityHasPrivilegesResponse); + it('should not search unauthorized indices', async () => { + mockCheckIndicesPrivileges.mockResolvedValueOnce({}); // empty object means no index is authorized const response = await server.inject(req, requestContextMock.convertContext(context)); expect(context.core.elasticsearch.client.asInternalUser.search).not.toHaveBeenCalled(); @@ -176,11 +199,19 @@ describe.skip('getResultsRoute route', () => { expect(response.body).toEqual([]); }); - it('handles pattern authorization error', async () => { + it('handles index discovery error', async () => { + const errorMessage = 'Error!'; + const mockGetIndices = context.core.elasticsearch.client.asInternalUser.indices.get; + mockGetIndices.mockRejectedValueOnce({ message: errorMessage }); + + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ message: errorMessage, status_code: 500 }); + }); + + it('handles index authorization error', async () => { const errorMessage = 'Error!'; - const mockHasPrivileges = - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; - mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage }); + mockCheckIndicesPrivileges.mockRejectedValueOnce({ message: errorMessage }); const response = await server.inject(req, requestContextMock.convertContext(context)); expect(response.status).toEqual(500); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts index 56729c7a40ab74..6c410e88f36268 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/get_results.ts @@ -11,20 +11,18 @@ import { RESULTS_ROUTE_PATH, INTERNAL_API_VERSION } from '../../../common/consta import { buildResponse } from '../../lib/build_response'; import { buildRouteValidation } from '../../schemas/common'; import { GetResultQuery } from '../../schemas/result'; -import type { Result, ResultDocument } from '../../schemas/result'; +import type { ResultDocument } from '../../schemas/result'; import { API_DEFAULT_ERROR_MESSAGE } from '../../translations'; import type { DataQualityDashboardRequestHandlerContext } from '../../types'; -import { createResultFromDocument } from './parser'; import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations'; +import { checkIndicesPrivileges } from './privileges'; -export const getQuery = (patterns: string[]) => ({ +export const getQuery = (indexName: string[]) => ({ size: 0, - query: { - bool: { filter: [{ terms: { 'rollup.pattern': patterns } }] }, - }, + query: { bool: { filter: [{ terms: { indexName } }] } }, aggs: { latest: { - terms: { field: 'rollup.pattern', size: 10000 }, // big enough to get all patterns, but under `index.max_terms_count` (default 65536) + terms: { field: 'indexName', size: 10000 }, // big enough to get all indexNames, but under `index.max_terms_count` (default 65536) aggs: { latest_doc: { top_hits: { size: 1, sort: [{ '@timestamp': { order: 'desc' } }] } } }, }, }, @@ -51,10 +49,6 @@ export const getResultsRoute = ( validate: { request: { query: buildRouteValidation(GetResultQuery) } }, }, async (context, request, response) => { - // TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 - return response.ok({ body: [] }); - - // eslint-disable-next-line no-unreachable const services = await context.resolve(['core', 'dataQualityDashboard']); const resp = buildResponse(response); @@ -70,38 +64,71 @@ export const getResultsRoute = ( } try { - // Confirm user has authorization for the requested patterns - const { patterns } = request.query; - const userEsClient = services.core.elasticsearch.client.asCurrentUser; - const privileges = await userEsClient.security.hasPrivileges({ - index: [ - { names: patterns.split(','), privileges: ['all', 'read', 'view_index_metadata'] }, - ], + const { client } = services.core.elasticsearch; + const { pattern } = request.query; + + // Discover all indices for the pattern using internal user + const indicesResponse = await client.asInternalUser.indices.get({ + index: pattern, + features: 'aliases', // omit 'settings' and 'mappings' to reduce response size }); - const authorizedPatterns = Object.keys(privileges.index).filter((pattern) => - Object.values(privileges.index[pattern]).some((v) => v === true) - ); - if (authorizedPatterns.length === 0) { + + // map data streams to their backing indices and collect indices to authorize + const indicesToAuthorize: string[] = []; + const dataStreamIndices: Record = {}; + Object.entries(indicesResponse).forEach(([indexName, { data_stream: dataStream }]) => { + if (dataStream) { + if (!dataStreamIndices[dataStream]) { + dataStreamIndices[dataStream] = []; + } + dataStreamIndices[dataStream].push(indexName); + } else { + indicesToAuthorize.push(indexName); + } + }); + indicesToAuthorize.push(...Object.keys(dataStreamIndices)); + if (indicesToAuthorize.length === 0) { return response.ok({ body: [] }); } - // Get the latest result of each pattern - const query = { index, ...getQuery(authorizedPatterns) }; - const internalEsClient = services.core.elasticsearch.client.asInternalUser; + // check privileges for indices or data streams + const hasIndexPrivileges = await checkIndicesPrivileges({ + client, + indices: indicesToAuthorize, + }); + + // filter out unauthorized indices, and expand data streams backing indices + const authorizedIndexNames = Object.entries(hasIndexPrivileges).reduce( + (acc, [indexName, authorized]) => { + if (authorized) { + if (dataStreamIndices[indexName]) { + acc.push(...dataStreamIndices[indexName]); + } else { + acc.push(indexName); + } + } + return acc; + }, + [] + ); + if (authorizedIndexNames.length === 0) { + return response.ok({ body: [] }); + } - const { aggregations } = await internalEsClient.search< + // Get the latest result for each indexName + const query = { index, ...getQuery(authorizedIndexNames) }; + const { aggregations } = await client.asInternalUser.search< ResultDocument, Record >(query); - const results: Result[] = - aggregations?.latest?.buckets.map((bucket) => - createResultFromDocument(bucket.latest_doc.hits.hits[0]._source) - ) ?? []; + const results: ResultDocument[] = + aggregations?.latest?.buckets.map((bucket) => bucket.latest_doc.hits.hits[0]._source) ?? + []; return response.ok({ body: results }); } catch (err) { - logger.error(JSON.stringify(err)); + logger.error(err.message); return resp.error({ body: err.message ?? API_DEFAULT_ERROR_MESSAGE, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.test.ts deleted file mode 100644 index 56800801ffc8f6..00000000000000 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { createDocumentFromResult, createResultFromDocument } from './parser'; -import { resultBody, resultDocument } from './results.mock'; - -describe('createDocumentFromResult', () => { - it('should create document from result', () => { - const document = createDocumentFromResult(resultBody); - expect(document).toEqual({ ...resultDocument, '@timestamp': expect.any(Number) }); - }); -}); - -describe('createResultFromDocument', () => { - it('should create document from result', () => { - const result = createResultFromDocument(resultDocument); - expect(result).toEqual({ ...resultBody, '@timestamp': expect.any(Number) }); - }); -}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.ts deleted file mode 100644 index 198d5522839e45..00000000000000 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/parser.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import type { Result, ResultDocument, IndexArray, IndexObject } from '../../schemas/result'; - -export const createDocumentFromResult = (result: Result): ResultDocument => { - const { rollup } = result; - const document: ResultDocument = { - ...result, - '@timestamp': Date.now(), - rollup: { - ...rollup, - ilmExplain: indexObjectToIndexArray(rollup.ilmExplain), - stats: indexObjectToIndexArray(rollup.stats), - results: indexObjectToIndexArray(rollup.results), - }, - }; - - return document; -}; - -export const createResultFromDocument = (document: ResultDocument): Result => { - const { rollup } = document; - const result = { - ...document, - rollup: { - ...rollup, - ilmExplain: indexArrayToIndexObject(rollup.ilmExplain), - stats: indexArrayToIndexObject(rollup.stats), - results: indexArrayToIndexObject(rollup.results), - }, - }; - - return result; -}; - -// ES parses object keys containing `.` as nested dot-separated field names (e.g. `event.name`). -// we need to convert documents containing objects with "indexName" keys (e.g. `.index-name-checked`) -// to object arrays so they can be stored correctly, we keep the key in the `_indexName` field. -const indexObjectToIndexArray = (obj: IndexObject): IndexArray => - Object.entries(obj).map(([key, value]) => ({ ...value, _indexName: key })); - -// convert index arrays back to objects with indexName as key -const indexArrayToIndexObject = (arr: IndexArray): IndexObject => - Object.fromEntries(arr.map(({ _indexName, ...value }) => [_indexName, value])); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts index 98eb67ecbaaa88..f3175a737ee54a 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.test.ts @@ -11,20 +11,29 @@ import { requestMock } from '../../__mocks__/request'; import { requestContextMock } from '../../__mocks__/request_context'; import { postResultsRoute } from './post_results'; import { loggerMock, type MockedLogger } from '@kbn/logging-mocks'; -import type { - SecurityHasPrivilegesResponse, - WriteResponseBase, -} from '@elastic/elasticsearch/lib/api/types'; -import { resultBody, resultDocument } from './results.mock'; - -// TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 -describe.skip('postResultsRoute route', () => { +import type { WriteResponseBase } from '@elastic/elasticsearch/lib/api/types'; +import { resultDocument } from './results.mock'; +import type { CheckIndicesPrivilegesParam } from './privileges'; + +const mockCheckIndicesPrivileges = jest.fn(({ indices }: CheckIndicesPrivilegesParam) => + Promise.resolve(Object.fromEntries(indices.map((index) => [index, true]))) +); +jest.mock('./privileges', () => ({ + checkIndicesPrivileges: (params: CheckIndicesPrivilegesParam) => + mockCheckIndicesPrivileges(params), +})); + +describe('postResultsRoute route', () => { describe('indexation', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); let logger: MockedLogger; - const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, body: resultBody }); + const req = requestMock.create({ + method: 'post', + path: RESULTS_ROUTE_PATH, + body: resultDocument, + }); beforeEach(() => { jest.clearAllMocks(); @@ -34,10 +43,9 @@ describe.skip('postResultsRoute route', () => { ({ context } = requestContextMock.createTools()); - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges.mockResolvedValue({ - has_all_requested: true, - } as unknown as SecurityHasPrivilegesResponse); - + context.core.elasticsearch.client.asInternalUser.indices.get.mockResolvedValue({ + [resultDocument.indexName]: {}, + }); postResultsRoute(server.router, logger); }); @@ -80,12 +88,16 @@ describe.skip('postResultsRoute route', () => { }); }); - describe('request pattern authorization', () => { + describe('request index authorization', () => { let server: ReturnType; let { context } = requestContextMock.createTools(); let logger: MockedLogger; - const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, body: resultBody }); + const req = requestMock.create({ + method: 'post', + path: RESULTS_ROUTE_PATH, + body: resultDocument, + }); beforeEach(() => { jest.clearAllMocks(); @@ -95,6 +107,9 @@ describe.skip('postResultsRoute route', () => { ({ context } = requestContextMock.createTools()); + context.core.elasticsearch.client.asInternalUser.indices.get.mockResolvedValue({ + [resultDocument.indexName]: {}, + }); context.core.elasticsearch.client.asInternalUser.index.mockResolvedValueOnce({ result: 'created', } as WriteResponseBase); @@ -102,42 +117,41 @@ describe.skip('postResultsRoute route', () => { postResultsRoute(server.router, logger); }); - it('should authorize pattern', async () => { - const mockHasPrivileges = - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; - mockHasPrivileges.mockResolvedValueOnce({ - has_all_requested: true, - } as unknown as SecurityHasPrivilegesResponse); + it('should authorize index', async () => { + const response = await server.inject(req, requestContextMock.convertContext(context)); + expect(mockCheckIndicesPrivileges).toHaveBeenCalledWith({ + client: context.core.elasticsearch.client, + indices: [resultDocument.indexName], + }); + expect(context.core.elasticsearch.client.asInternalUser.index).toHaveBeenCalled(); + expect(response.status).toEqual(200); + expect(response.body).toEqual({ result: 'created' }); + }); + + it('should authorize data stream', async () => { + const dataStreamName = 'test_data_stream_name'; + context.core.elasticsearch.client.asInternalUser.indices.get.mockResolvedValue({ + [resultDocument.indexName]: { data_stream: dataStreamName }, + }); + mockCheckIndicesPrivileges.mockResolvedValueOnce({ [dataStreamName]: true }); const response = await server.inject(req, requestContextMock.convertContext(context)); - expect(mockHasPrivileges).toHaveBeenCalledWith({ - index: [ - { - names: [resultBody.rollup.pattern], - privileges: ['all', 'read', 'view_index_metadata'], - }, - ], + expect(mockCheckIndicesPrivileges).toHaveBeenCalledWith({ + client: context.core.elasticsearch.client, + indices: [dataStreamName], }); expect(context.core.elasticsearch.client.asInternalUser.index).toHaveBeenCalled(); expect(response.status).toEqual(200); expect(response.body).toEqual({ result: 'created' }); }); - it('should not index unauthorized pattern', async () => { - const mockHasPrivileges = - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; - mockHasPrivileges.mockResolvedValueOnce({ - has_all_requested: false, - } as unknown as SecurityHasPrivilegesResponse); + it('should not index unauthorized index', async () => { + mockCheckIndicesPrivileges.mockResolvedValueOnce({ [resultDocument.indexName]: false }); const response = await server.inject(req, requestContextMock.convertContext(context)); - expect(mockHasPrivileges).toHaveBeenCalledWith({ - index: [ - { - names: [resultBody.rollup.pattern], - privileges: ['all', 'read', 'view_index_metadata'], - }, - ], + expect(mockCheckIndicesPrivileges).toHaveBeenCalledWith({ + client: context.core.elasticsearch.client, + indices: [resultDocument.indexName], }); expect(context.core.elasticsearch.client.asInternalUser.index).not.toHaveBeenCalled(); @@ -145,11 +159,9 @@ describe.skip('postResultsRoute route', () => { expect(response.body).toEqual({ result: 'noop' }); }); - it('handles pattern authorization error', async () => { + it('handles index authorization error', async () => { const errorMessage = 'Error!'; - const mockHasPrivileges = - context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; - mockHasPrivileges.mockRejectedValueOnce({ message: errorMessage }); + mockCheckIndicesPrivileges.mockRejectedValueOnce(Error(errorMessage)); const response = await server.inject(req, requestContextMock.convertContext(context)); expect(response.status).toEqual(500); @@ -170,7 +182,7 @@ describe.skip('postResultsRoute route', () => { const req = requestMock.create({ method: 'post', path: RESULTS_ROUTE_PATH, - body: { rollup: resultBody.rollup }, + body: { indexName: 'invalid body' }, }); const result = server.validate(req); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.ts index 1162d23f1dfad7..b4b2e4b219bc40 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/post_results.ts @@ -13,7 +13,7 @@ import { buildRouteValidation } from '../../schemas/common'; import { PostResultBody } from '../../schemas/result'; import { API_DEFAULT_ERROR_MESSAGE } from '../../translations'; import type { DataQualityDashboardRequestHandlerContext } from '../../types'; -import { createDocumentFromResult } from './parser'; +import { checkIndicesPrivileges } from './privileges'; import { API_RESULTS_INDEX_NOT_AVAILABLE } from './translations'; export const postResultsRoute = ( @@ -32,10 +32,6 @@ export const postResultsRoute = ( validate: { request: { body: buildRouteValidation(PostResultBody) } }, }, async (context, request, response) => { - // TODO: https://github.com/elastic/kibana/pull/173185#issuecomment-1908034302 - return response.ok({ body: { result: 'noop' } }); - - // eslint-disable-next-line no-unreachable const services = await context.resolve(['core', 'dataQualityDashboard']); const resp = buildResponse(response); @@ -51,24 +47,35 @@ export const postResultsRoute = ( } try { - // Confirm user has authorization for the pattern payload - const { pattern } = request.body.rollup; - const userEsClient = services.core.elasticsearch.client.asCurrentUser; - const privileges = await userEsClient.security.hasPrivileges({ - index: [{ names: [pattern], privileges: ['all', 'read', 'view_index_metadata'] }], + const { client } = services.core.elasticsearch; + const { indexName } = request.body; + + // Confirm index exists and get the data stream name if it's a data stream + const indicesResponse = await client.asInternalUser.indices.get({ + index: indexName, + features: 'aliases', + }); + if (!indicesResponse[indexName]) { + return response.ok({ body: { result: 'noop' } }); + } + const indexOrDataStream = indicesResponse[indexName].data_stream ?? indexName; + + // Confirm user has authorization for the index name or data stream + const hasIndexPrivileges = await checkIndicesPrivileges({ + client, + indices: [indexOrDataStream], }); - if (!privileges.has_all_requested) { + if (!hasIndexPrivileges[indexOrDataStream]) { return response.ok({ body: { result: 'noop' } }); } // Index the result - const document = createDocumentFromResult(request.body); - const esClient = services.core.elasticsearch.client.asInternalUser; - const outcome = await esClient.index({ index, body: document }); + const body = { '@timestamp': Date.now(), ...request.body }; + const outcome = await client.asInternalUser.index({ index, body }); return response.ok({ body: { result: outcome.result } }); } catch (err) { - logger.error(JSON.stringify(err)); + logger.error(err.message); return resp.error({ body: err.message ?? API_DEFAULT_ERROR_MESSAGE, diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/privileges.test.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/privileges.test.ts new file mode 100644 index 00000000000000..2833e3f030fdd5 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/privileges.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SecurityHasPrivilegesResponse } from '@elastic/elasticsearch/lib/api/types'; +import { requestContextMock } from '../../__mocks__/request_context'; +import { checkIndicesPrivileges } from './privileges'; + +// const mockHasPrivileges = +// context.core.elasticsearch.client.asCurrentUser.security.hasPrivileges; +// mockHasPrivileges.mockResolvedValueOnce({ +// has_all_requested: true, +// } as unknown as SecurityHasPrivilegesResponse); + +describe('checkIndicesPrivileges', () => { + const { context } = requestContextMock.createTools(); + const { client } = context.core.elasticsearch; + + beforeEach(() => { + client.asCurrentUser.security.hasPrivileges.mockResolvedValue({ + index: { + index1: { + read: true, + view_index_metadata: true, + manage: true, + monitor: true, + }, + index2: { + read: true, + view_index_metadata: true, + manage: true, + monitor: true, + }, + }, + } as unknown as SecurityHasPrivilegesResponse); + }); + + it('should return true if user has required privileges', async () => { + const result = await checkIndicesPrivileges({ client, indices: ['index1', 'index2'] }); + expect(result).toEqual({ index1: true, index2: true }); + }); + + it('should return true if only monitor privileges is missing', async () => { + client.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce({ + index: { + index1: { + read: true, + view_index_metadata: true, + manage: true, + monitor: false, + }, + }, + } as unknown as SecurityHasPrivilegesResponse); + const result = await checkIndicesPrivileges({ client, indices: ['index1'] }); + + expect(result).toEqual({ index1: true }); + }); + + it('should return true if only manage privileges is missing', async () => { + client.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce({ + index: { + index1: { + read: true, + view_index_metadata: true, + manage: false, + monitor: true, + }, + }, + } as unknown as SecurityHasPrivilegesResponse); + + const result = await checkIndicesPrivileges({ client, indices: ['index1'] }); + + expect(result).toEqual({ index1: true }); + }); + + it('should return false if both manage and monitor privileges is missing', async () => { + client.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce({ + index: { + index1: { + read: true, + view_index_metadata: true, + manage: false, + monitor: false, + }, + }, + } as unknown as SecurityHasPrivilegesResponse); + + const result = await checkIndicesPrivileges({ client, indices: ['index1'] }); + + expect(result).toEqual({ index1: false }); + }); + + it('should return false if only read privilege is missing', async () => { + client.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce({ + index: { + index1: { + read: false, + view_index_metadata: true, + manage: true, + monitor: true, + }, + }, + } as unknown as SecurityHasPrivilegesResponse); + + const result = await checkIndicesPrivileges({ client, indices: ['index1'] }); + + expect(result).toEqual({ index1: false }); + }); + + it('should return false if only view_index_metadata privilege is missing', async () => { + client.asCurrentUser.security.hasPrivileges.mockResolvedValueOnce({ + index: { + index1: { + read: true, + view_index_metadata: false, + manage: true, + monitor: true, + }, + }, + } as unknown as SecurityHasPrivilegesResponse); + + const result = await checkIndicesPrivileges({ client, indices: ['index1'] }); + + expect(result).toEqual({ index1: false }); + }); +}); diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/privileges.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/privileges.ts new file mode 100644 index 00000000000000..ebda2f54e16e04 --- /dev/null +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/privileges.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { IScopedClusterClient } from '@kbn/core-elasticsearch-server'; + +export interface CheckIndicesPrivilegesParam { + client: IScopedClusterClient; + indices: string[]; +} + +/** + * Checks user has the required privileges to do a results check for the given indices. + * In order to be allowed to do a result check user needs: + * `read`, `view_index_metadata` and (`monitor` or `manage`) index privileges. + */ +export const checkIndicesPrivileges = async ({ client, indices }: CheckIndicesPrivilegesParam) => { + const privileges = await client.asCurrentUser.security.hasPrivileges({ + index: [{ names: indices, privileges: ['read', 'view_index_metadata', 'monitor', 'manage'] }], + }); + + const hasRequiredIndexPrivilege: Record = {}; + Object.entries(privileges.index).forEach( + ([indexName, { read, view_index_metadata: viewMetadata, monitor, manage }]) => { + hasRequiredIndexPrivilege[indexName] = read && viewMetadata && (monitor || manage); + } + ); + + return hasRequiredIndexPrivilege; +}; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts index 1d0b15a4c24c01..36ca3d2dc4e664 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/routes/results/results.mock.ts @@ -8,195 +8,29 @@ import type { ResultDocument } from '../../schemas/result'; export const resultDocument: ResultDocument = { - '@timestamp': 1622767273955, - meta: { - batchId: 'aae36cd8-3825-4ad1-baa4-79bdf4617f8a', - ecsVersion: '8.6.1', - errorCount: 0, - ilmPhase: 'hot', - indexId: 'aO29KOwtQ3Snf-Pit5Wf4w', - indexName: '.internal.alerts-security.alerts-default-000001', - isCheckAll: true, - numberOfDocuments: 20, - numberOfFields: 1726, - numberOfIncompatibleFields: 2, - numberOfEcsFields: 1440, - numberOfCustomFields: 284, - numberOfIndices: 1, - numberOfIndicesChecked: 1, - numberOfSameFamily: 0, - sameFamilyFields: [], - sizeInBytes: 506471, - timeConsumedMs: 85, - unallowedMappingFields: [], - unallowedValueFields: ['event.category', 'event.outcome'], - }, - rollup: { - docsCount: 20, - error: null, - ilmExplain: [ - { - _indexName: '.internal.alerts-security.alerts-default-000001', - index: '.internal.alerts-security.alerts-default-000001', - managed: true, - policy: '.alerts-ilm-policy', - index_creation_date_millis: 1700757268526, - time_since_index_creation: '20.99d', - lifecycle_date_millis: 1700757268526, - age: '20.99d', - phase: 'hot', - phase_time_millis: 1700757270294, - action: 'rollover', - action_time_millis: 1700757273955, - step: 'check-rollover-ready', - step_time_millis: 1700757273955, - phase_execution: { - policy: '.alerts-ilm-policy', - phase_definition: { - min_age: '0ms', - actions: { - rollover: { - max_age: '30d', - max_primary_shard_size: '50gb', - }, - }, - }, - version: 1, - modified_date_in_millis: 1700757266723, - }, - }, - ], - ilmExplainPhaseCounts: { - hot: 1, - warm: 0, - cold: 0, - frozen: 0, - unmanaged: 0, - }, - indices: 1, - pattern: '.alerts-security.alerts-default', - results: [ - { - _indexName: '.internal.alerts-security.alerts-default-000001', - docsCount: 20, - error: null, - ilmPhase: 'hot', - incompatible: 2, - indexName: '.internal.alerts-security.alerts-default-000001', - markdownComments: [ - '### .internal.alerts-security.alerts-default-000001\n', - '| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .internal.alerts-security.alerts-default-000001 | 20 (100,0 %) | 2 | `hot` | 494.6KB |\n\n', - '### **Incompatible fields** `2` **Same family** `0` **Custom fields** `284` **ECS compliant fields** `1440` **All fields** `1726`\n', - "#### 2 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n", - '\n\n#### Incompatible field values - .internal.alerts-security.alerts-default-000001\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `behavior` (1) |\n| event.outcome | `failure`, `success`, `unknown` | `` (12) |\n\n', - ], - pattern: '.alerts-security.alerts-default', - sameFamily: 0, - }, - ], - sizeInBytes: 506471, - stats: [ - { - _indexName: '.internal.alerts-security.alerts-default-000001', - uuid: 'aO29KOwtQ3Snf-Pit5Wf4w', - health: 'green', - status: 'open', - }, - ], - }, -}; - -export const resultBody = { - meta: { - batchId: 'aae36cd8-3825-4ad1-baa4-79bdf4617f8a', - ecsVersion: '8.6.1', - errorCount: 0, - ilmPhase: 'hot', - indexId: 'aO29KOwtQ3Snf-Pit5Wf4w', - indexName: '.internal.alerts-security.alerts-default-000001', - isCheckAll: true, - numberOfDocuments: 20, - numberOfFields: 1726, - numberOfIncompatibleFields: 2, - numberOfEcsFields: 1440, - numberOfCustomFields: 284, - numberOfIndices: 1, - numberOfIndicesChecked: 1, - numberOfSameFamily: 0, - sameFamilyFields: [], - sizeInBytes: 506471, - timeConsumedMs: 85, - unallowedMappingFields: [], - unallowedValueFields: ['event.category', 'event.outcome'], - }, - rollup: { - docsCount: 20, - error: null, - ilmExplain: { - '.internal.alerts-security.alerts-default-000001': { - index: '.internal.alerts-security.alerts-default-000001', - managed: true, - policy: '.alerts-ilm-policy', - index_creation_date_millis: 1700757268526, - time_since_index_creation: '20.99d', - lifecycle_date_millis: 1700757268526, - age: '20.99d', - phase: 'hot', - phase_time_millis: 1700757270294, - action: 'rollover', - action_time_millis: 1700757273955, - step: 'check-rollover-ready', - step_time_millis: 1700757273955, - phase_execution: { - policy: '.alerts-ilm-policy', - phase_definition: { - min_age: '0ms', - actions: { - rollover: { - max_age: '30d', - max_primary_shard_size: '50gb', - }, - }, - }, - version: 1, - modified_date_in_millis: 1700757266723, - }, - }, - }, - ilmExplainPhaseCounts: { - hot: 1, - warm: 0, - cold: 0, - frozen: 0, - unmanaged: 0, - }, - indices: 1, - pattern: '.alerts-security.alerts-default', - results: { - '.internal.alerts-security.alerts-default-000001': { - docsCount: 20, - error: null, - ilmPhase: 'hot', - incompatible: 2, - indexName: '.internal.alerts-security.alerts-default-000001', - markdownComments: [ - '### .internal.alerts-security.alerts-default-000001\n', - '| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .internal.alerts-security.alerts-default-000001 | 20 (100,0 %) | 2 | `hot` | 494.6KB |\n\n', - '### **Incompatible fields** `2` **Same family** `0` **Custom fields** `284` **ECS compliant fields** `1440` **All fields** `1726`\n', - "#### 2 incompatible fields\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n", - '\n\n#### Incompatible field values - .internal.alerts-security.alerts-default-000001\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `behavior` (1) |\n| event.outcome | `failure`, `success`, `unknown` | `` (12) |\n\n', - ], - pattern: '.alerts-security.alerts-default', - sameFamily: 0, - }, - }, - sizeInBytes: 506471, - stats: { - '.internal.alerts-security.alerts-default-000001': { - uuid: 'aO29KOwtQ3Snf-Pit5Wf4w', - health: 'green', - status: 'open', - }, - }, - }, + batchId: '33d95427-1fd3-43c3-bdeb-74324533a31e', + indexName: '.ds-logs-endpoint.alerts-default-2023.11.23-000001', + isCheckAll: false, + checkedAt: 1706526408000, + docsCount: 100, + totalFieldCount: 1582, + ecsFieldCount: 677, + customFieldCount: 904, + incompatibleFieldCount: 1, + sameFamilyFieldCount: 0, + sameFamilyFields: [], + unallowedMappingFields: [], + unallowedValueFields: ['event.category'], + sizeInBytes: 173796, + ilmPhase: 'hot', + markdownComments: [ + '### .ds-logs-endpoint.alerts-default-2023.11.23-000001\n', + '| Result | Index | Docs | Incompatible fields | ILM Phase | Size |\n|--------|-------|------|---------------------|-----------|------|\n| ❌ | .ds-logs-endpoint.alerts-default-2023.11.23-000001 | 100 (64,1 %) | 1 | `hot` | 274.6KB |\n\n', + '### **Incompatible fields** `1` **Same family** `0` **Custom fields** `904` **ECS compliant fields** `677` **All fields** `1582`\n', + "#### 1 incompatible field\n\nFields are incompatible with ECS when index mappings, or the values of the fields in the index, don't conform to the Elastic Common Schema (ECS), version 8.6.1.\n\n❌ Detection engine rules referencing these fields may not match them correctly\n❌ Pages may not display some events or fields due to unexpected field mappings or values\n❌ Mappings or field values that don't comply with ECS are not supported\n", + '\n\n#### Incompatible field values - .ds-logs-endpoint.alerts-default-2023.11.23-000001\n\n\n| Field | ECS values (expected) | Document values (actual) | \n|-------|-----------------------|--------------------------|\n| event.category | `authentication`, `configuration`, `database`, `driver`, `email`, `file`, `host`, `iam`, `intrusion_detection`, `malware`, `network`, `package`, `process`, `registry`, `session`, `threat`, `vulnerability`, `web` | `behavior` (6) |\n\n', + ], + ecsVersion: '8.6.1', + indexId: 'PMhntcuPQ_yhPoNsXiM_hg', + error: null, }; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts index 09851c9b8dc86c..69387ea6fd8cf8 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts +++ b/x-pack/plugins/ecs_data_quality_dashboard/server/schemas/result.ts @@ -7,64 +7,30 @@ import * as t from 'io-ts'; -export const ResultMeta = t.type({ +export const ResultDocument = t.type({ batchId: t.string, - ecsVersion: t.string, - errorCount: t.number, - ilmPhase: t.string, - indexId: t.string, indexName: t.string, isCheckAll: t.boolean, - numberOfDocuments: t.number, - numberOfFields: t.number, - numberOfIncompatibleFields: t.number, - numberOfEcsFields: t.number, - numberOfCustomFields: t.number, - numberOfIndices: t.number, - numberOfIndicesChecked: t.number, - numberOfSameFamily: t.number, + checkedAt: t.number, + docsCount: t.number, + totalFieldCount: t.number, + ecsFieldCount: t.number, + customFieldCount: t.number, + incompatibleFieldCount: t.number, + sameFamilyFieldCount: t.number, sameFamilyFields: t.array(t.string), - sizeInBytes: t.number, - timeConsumedMs: t.number, unallowedMappingFields: t.array(t.string), unallowedValueFields: t.array(t.string), -}); -export type ResultMeta = t.TypeOf; - -export const ResultRollup = t.type({ - docsCount: t.number, - error: t.union([t.string, t.null]), - indices: t.number, - pattern: t.string, sizeInBytes: t.number, - ilmExplainPhaseCounts: t.record(t.string, t.number), - ilmExplain: t.record(t.string, t.UnknownRecord), - stats: t.record(t.string, t.UnknownRecord), - results: t.record(t.string, t.UnknownRecord), -}); -export type ResultRollup = t.TypeOf; - -export const Result = t.type({ - meta: ResultMeta, - rollup: ResultRollup, + ilmPhase: t.string, + markdownComments: t.array(t.string), + ecsVersion: t.string, + indexId: t.string, + error: t.union([t.string, t.null]), }); -export type Result = t.TypeOf; - -export type IndexArray = Array<{ _indexName: string } & Record>; -export type IndexObject = Record>; +export type ResultDocument = t.TypeOf; -export type ResultDocument = Omit & { - '@timestamp': number; - rollup: Omit & { - stats: IndexArray; - results: IndexArray; - ilmExplain: IndexArray; - }; -}; +export const PostResultBody = ResultDocument; -// Routes validation schemas - -export const GetResultQuery = t.type({ patterns: t.string }); +export const GetResultQuery = t.type({ pattern: t.string }); export type GetResultQuery = t.TypeOf; - -export const PostResultBody = Result; diff --git a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json index 04a7d2bf092f55..b725beec802b27 100644 --- a/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json +++ b/x-pack/plugins/ecs_data_quality_dashboard/tsconfig.json @@ -24,6 +24,7 @@ "@kbn/data-stream-adapter", "@kbn/spaces-plugin", "@kbn/core-elasticsearch-server-mocks", + "@kbn/core-elasticsearch-server", ], "exclude": [ "target/**/*",