Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Security Solution] Data Quality Dashboard persistence #175673

Merged
merged 19 commits into from
Feb 6, 2024
Merged

[Security Solution] Data Quality Dashboard persistence #175673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
c06d4f2
storage improvements
semd Jan 26, 2024
5e8111e
update codeowners
semd Jan 26, 2024
1fb9b20
Merge branch 'main' into 7382_data_quality_dashboard_persistence
kibanamachine Jan 26, 2024
9dad05d
[CI] Auto-commit changed files from 'node scripts/generate codeowners'
kibanamachine Jan 26, 2024
d8f8cf1
remove parser
semd Jan 26, 2024
a759810
default value for tests
semd Jan 29, 2024
984f6d0
Merge branch 'main' into 7382_data_quality_dashboard_persistence
kibanamachine Jan 29, 2024
199ea53
add last check date column
semd Jan 29, 2024
db552d0
update mocks
semd Jan 29, 2024
ce501a6
mocks and test
semd Jan 29, 2024
b726ecc
package ownership
semd Jan 29, 2024
a4d0bfe
Merge branch 'main' into 7382_data_quality_dashboard_persistence
kibanamachine Jan 29, 2024
d189669
Merge branch 'main' into 7382_data_quality_dashboard_persistence
kibanamachine Jan 29, 2024
e0be9c9
fix privileges and last check issue
semd Jan 30, 2024
d611a3f
[CI] Auto-commit changed files from 'node scripts/lint_ts_projects --…
kibanamachine Jan 30, 2024
6690d7b
change error loging
semd Jan 30, 2024
2ceee6e
fix data stream authorization
semd Feb 1, 2024
482bfc0
Merge branch 'main' into 7382_data_quality_dashboard_persistence
kibanamachine Feb 1, 2024
49ee47e
Merge branch 'main' into 7382_data_quality_dashboard_persistence
kibanamachine Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/CODEOWNERS
View file
Open in desktop
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -353,8 +353,8 @@ packages/kbn-docs-utils @elastic/kibana-operations
packages/kbn-dom-drag-drop @elastic/kibana-visualizations @elastic/kibana-data-discovery
packages/kbn-ebt-tools @elastic/kibana-core
packages/kbn-ecs @elastic/kibana-core @elastic/security-threat-hunting-investigations
x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore
x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore
packages/kbn-elastic-agent-utils @elastic/obs-ux-logs-team
x-pack/packages/kbn-elastic-assistant @elastic/security-generative-ai
x-pack/packages/kbn-elastic-assistant-common @elastic/security-generative-ai
Expand Down
16 changes: 11 additions & 5 deletions ...cs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import { useAddToNewCase } from '../../use_add_to_new_case';
import { useMappings } from '../../use_mappings';
import { useUnallowedValues } from '../../use_unallowed_values';
import { useDataQualityContext } from '../data_quality_context';
import { getSizeInBytes, postResult } from '../../helpers';
import { formatStorageResult, postStorageResult, getSizeInBytes } from '../../helpers';

const EMPTY_MARKDOWN_COMMENTS: string[] = [];

Expand Down Expand Up @@ -249,6 +249,8 @@ const IndexPropertiesComponent: React.FC<Props> = ({
})
: EMPTY_MARKDOWN_COMMENTS;

const checkedAt = partitionedFieldMetadata ? Date.now() : undefined;

const updatedRollup = {
...patternRollup,
results: {
Expand All @@ -262,13 +264,14 @@ const IndexPropertiesComponent: React.FC<Props> = ({
markdownComments,
pattern,
sameFamily: indexSameFamily,
checkedAt,
},
},
};
updatePatternRollup(updatedRollup);

if (indexId && requestTime != null && requestTime > 0 && partitionedFieldMetadata) {
const checkMetadata = {
const report = {
batchId: uuidv4(),
ecsVersion: EcsVersion,
errorCount: error ? 1 : 0,
Expand All @@ -294,10 +297,13 @@ const IndexPropertiesComponent: React.FC<Props> = ({
partitionedFieldMetadata.incompatible
),
};
telemetryEvents.reportDataQualityIndexChecked?.(checkMetadata);
telemetryEvents.reportDataQualityIndexChecked?.(report);

const result = { meta: checkMetadata, rollup: updatedRollup };
postResult({ result, httpFetch, toasts, abortController: new AbortController() });
const result = updatedRollup.results[indexName];
if (result) {
const storageResult = formatStorageResult({ result, report, partitionedFieldMetadata });
postStorageResult({ storageResult, httpFetch, toasts });
}
}
}
}
Expand Down
39 changes: 39 additions & 0 deletions ...n/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,7 @@ describe('helpers', () => {
],
pattern: 'auditbeat-*',
sameFamily: 0,
checkedAt: 1706526408000,
},
};
const isILMAvailable = true;
Expand All @@ -300,6 +301,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 733175040,
checkedAt: undefined,
},
{
docsCount: 1628343,
Expand All @@ -309,6 +311,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 731583142,
checkedAt: undefined,
},
{
docsCount: 4,
Expand All @@ -318,6 +321,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 28413,
checkedAt: 1706526408000,
},
]);
});
Expand All @@ -344,6 +348,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 733175040,
checkedAt: undefined,
},
{
docsCount: 1628343,
Expand All @@ -353,6 +358,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 731583142,
checkedAt: undefined,
},
{
docsCount: 4,
Expand All @@ -362,6 +368,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 28413,
checkedAt: 1706526408000,
},
]);
});
Expand All @@ -388,6 +395,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 28413,
checkedAt: 1706526408000,
},
{
docsCount: 1628343,
Expand All @@ -397,6 +405,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 731583142,
checkedAt: undefined,
},
{
docsCount: 1630289,
Expand All @@ -406,6 +415,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 733175040,
checkedAt: undefined,
},
]);
});
Expand All @@ -432,6 +442,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 0,
checkedAt: undefined,
},
{
docsCount: 0,
Expand All @@ -441,6 +452,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 0,
checkedAt: undefined,
},
{
docsCount: 0,
Expand All @@ -450,6 +462,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 4,
sizeInBytes: 0,
checkedAt: undefined,
},
]);
});
Expand Down Expand Up @@ -701,6 +714,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 43357342,
checkedAt: 1706526408000,
},
{
docsCount: 48068,
Expand All @@ -710,6 +724,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 32460397,
checkedAt: 1706526408000,
},
{
docsCount: 48064,
Expand All @@ -719,6 +734,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 42782794,
checkedAt: 1706526408000,
},
{
docsCount: 47868,
Expand All @@ -728,6 +744,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 31575964,
checkedAt: 1706526408000,
},
{
docsCount: 47827,
Expand All @@ -737,6 +754,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 44130657,
checkedAt: 1706526408000,
},
{
docsCount: 47642,
Expand All @@ -746,6 +764,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 42412521,
checkedAt: 1706526408000,
},
{
docsCount: 47545,
Expand All @@ -755,6 +774,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 41423244,
checkedAt: 1706526408000,
},
{
docsCount: 47531,
Expand All @@ -764,6 +784,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 32394133,
checkedAt: 1706526408000,
},
{
docsCount: 47530,
Expand All @@ -773,6 +794,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 43015519,
checkedAt: 1706526408000,
},
{
docsCount: 47520,
Expand All @@ -782,6 +804,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 42230604,
checkedAt: 1706526408000,
},
{
docsCount: 47496,
Expand All @@ -791,6 +814,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 41710968,
checkedAt: 1706526408000,
},
{
docsCount: 47486,
Expand All @@ -800,6 +824,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 42295944,
checkedAt: 1706526408000,
},
{
docsCount: 47486,
Expand All @@ -809,6 +834,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 41761321,
checkedAt: 1706526408000,
},
{
docsCount: 47460,
Expand All @@ -818,6 +844,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 30481198,
checkedAt: 1706526408000,
},
{
docsCount: 47439,
Expand All @@ -827,6 +854,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 41554041,
checkedAt: 1706526408000,
},
{
docsCount: 47395,
Expand All @@ -836,6 +864,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 42815907,
checkedAt: 1706526408000,
},
{
docsCount: 47394,
Expand All @@ -845,6 +874,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 41157112,
checkedAt: 1706526408000,
},
{
docsCount: 47372,
Expand All @@ -854,6 +884,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 31626792,
checkedAt: 1706526408000,
},
{
docsCount: 47369,
Expand All @@ -863,6 +894,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 41828969,
checkedAt: 1706526408000,
},
{
docsCount: 47348,
Expand All @@ -872,6 +904,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 40010773,
checkedAt: 1706526408000,
},
{
docsCount: 47339,
Expand All @@ -881,6 +914,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 43480570,
checkedAt: 1706526408000,
},
{
docsCount: 47325,
Expand All @@ -890,6 +924,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 41822475,
checkedAt: 1706526408000,
},
{
docsCount: 47294,
Expand All @@ -899,6 +934,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 43018490,
checkedAt: 1706526408000,
},
{
docsCount: 24276,
Expand All @@ -908,6 +944,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 23579440,
checkedAt: 1706526408000,
},
{
docsCount: 4,
Expand All @@ -917,6 +954,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 28409,
checkedAt: 1706526408000,
},
{
docsCount: 0,
Expand All @@ -926,6 +964,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 1118155,
sizeInBytes: 247,
checkedAt: 1706526408000,
},
],
pageSize: 10,
Expand Down
1 change: 1 addition & 0 deletions ...lution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ export const getSummaryTableItems = ({
pattern,
patternDocsCount,
sizeInBytes: getSizeInBytes({ stats, indexName }),
checkedAt: results?.[indexName]?.checkedAt,
}));

return orderBy([sortByColumn], [sortByDirection], summaryTableItems);
Expand Down
2 changes: 2 additions & 0 deletions ...ata_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ describe('helpers', () => {
pattern: 'auditbeat-*',
patternDocsCount: 57410,
sizeInBytes: 103344068,
checkedAt: Date.now(),
};

const hasIncompatible: IndexSummaryTableItem = {
Expand Down Expand Up @@ -188,6 +189,7 @@ describe('helpers', () => {
},
{ field: 'ilmPhase', name: 'ILM Phase', sortable: true, truncateText: false },
{ field: 'sizeInBytes', name: 'Size', sortable: true, truncateText: false },
{ field: 'checkedAt', name: 'Last check', sortable: true, truncateText: false },
]);
});

Expand Down
Loading