[Security Solution] Data Quality Dashboard persistence (elastic#175673)

## Summary follow-up of elastic#173185 This PR enables the persistence layer implemented in the previous PR, applying the following changes: - Update the mapping to store unitary index results instead of storing the whole pattern with the results in each document. - Change the query to get the stored results by aggregating documents by indexName. The authorized indexNames derived from the `pattern` parameter are retrieved using the `indices.get` request. - A bug involving a race condition with the initialization and the retrieval of stored results, resulting in an unintended reset of the results in the UI, has been fixed. https://github.com/elastic/kibana/assets/17747913/0598606b-c5f4-42b3-901c-f86a3cac65e4 --------- Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
CoenWarmer · Feb 15, 2024 · 6f1eb03 · 6f1eb03
1 parent 4f15ef3
commit 6f1eb03
Show file tree

Hide file tree

Showing 35 changed files with 712 additions and 564 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -353,8 +353,8 @@ packages/kbn-docs-utils @elastic/kibana-operations
 packages/kbn-dom-drag-drop @elastic/kibana-visualizations @elastic/kibana-data-discovery
 packages/kbn-ebt-tools @elastic/kibana-core
 packages/kbn-ecs @elastic/kibana-core @elastic/security-threat-hunting-investigations
-x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
-x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
+x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore
+x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore
 packages/kbn-elastic-agent-utils @elastic/obs-ux-logs-team
 x-pack/packages/kbn-elastic-assistant @elastic/security-generative-ai
 x-pack/packages/kbn-elastic-assistant-common @elastic/security-generative-ai

diff --git a/...cs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx b/...cs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx
@@ -44,7 +44,7 @@ import { useAddToNewCase } from '../../use_add_to_new_case';
 import { useMappings } from '../../use_mappings';
 import { useUnallowedValues } from '../../use_unallowed_values';
 import { useDataQualityContext } from '../data_quality_context';
-import { getSizeInBytes, postResult } from '../../helpers';
+import { formatStorageResult, postStorageResult, getSizeInBytes } from '../../helpers';
 
 const EMPTY_MARKDOWN_COMMENTS: string[] = [];
 
@@ -249,6 +249,8 @@ const IndexPropertiesComponent: React.FC<Props> = ({
               })
             : EMPTY_MARKDOWN_COMMENTS;
 
+        const checkedAt = partitionedFieldMetadata ? Date.now() : undefined;
+
         const updatedRollup = {
           ...patternRollup,
           results: {
@@ -262,13 +264,14 @@ const IndexPropertiesComponent: React.FC<Props> = ({
               markdownComments,
               pattern,
               sameFamily: indexSameFamily,
+              checkedAt,
             },
           },
         };
         updatePatternRollup(updatedRollup);
 
         if (indexId && requestTime != null && requestTime > 0 && partitionedFieldMetadata) {
-          const checkMetadata = {
+          const report = {
             batchId: uuidv4(),
             ecsVersion: EcsVersion,
             errorCount: error ? 1 : 0,
@@ -294,10 +297,13 @@ const IndexPropertiesComponent: React.FC<Props> = ({
               partitionedFieldMetadata.incompatible
             ),
           };
-          telemetryEvents.reportDataQualityIndexChecked?.(checkMetadata);
+          telemetryEvents.reportDataQualityIndexChecked?.(report);
 
-          const result = { meta: checkMetadata, rollup: updatedRollup };
-          postResult({ result, httpFetch, toasts, abortController: new AbortController() });
+          const result = updatedRollup.results[indexName];
+          if (result) {
+            const storageResult = formatStorageResult({ result, report, partitionedFieldMetadata });
+            postStorageResult({ storageResult, httpFetch, toasts });
+          }
         }
       }
     }

diff --git a/...n/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts b/...n/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.test.ts
@@ -274,6 +274,7 @@ describe('helpers', () => {
         ],
         pattern: 'auditbeat-*',
         sameFamily: 0,
+        checkedAt: 1706526408000,
       },
     };
     const isILMAvailable = true;
@@ -300,6 +301,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 733175040,
+          checkedAt: undefined,
         },
         {
           docsCount: 1628343,
@@ -309,6 +311,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 731583142,
+          checkedAt: undefined,
         },
         {
           docsCount: 4,
@@ -318,6 +321,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 28413,
+          checkedAt: 1706526408000,
         },
       ]);
     });
@@ -344,6 +348,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 733175040,
+          checkedAt: undefined,
         },
         {
           docsCount: 1628343,
@@ -353,6 +358,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 731583142,
+          checkedAt: undefined,
         },
         {
           docsCount: 4,
@@ -362,6 +368,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 28413,
+          checkedAt: 1706526408000,
         },
       ]);
     });
@@ -388,6 +395,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 28413,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 1628343,
@@ -397,6 +405,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 731583142,
+          checkedAt: undefined,
         },
         {
           docsCount: 1630289,
@@ -406,6 +415,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 733175040,
+          checkedAt: undefined,
         },
       ]);
     });
@@ -432,6 +442,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 0,
+          checkedAt: undefined,
         },
         {
           docsCount: 0,
@@ -441,6 +452,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 0,
+          checkedAt: undefined,
         },
         {
           docsCount: 0,
@@ -450,6 +462,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 4,
           sizeInBytes: 0,
+          checkedAt: undefined,
         },
       ]);
     });
@@ -701,6 +714,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 43357342,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 48068,
@@ -710,6 +724,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 32460397,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 48064,
@@ -719,6 +734,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 42782794,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47868,
@@ -728,6 +744,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 31575964,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47827,
@@ -737,6 +754,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 44130657,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47642,
@@ -746,6 +764,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 42412521,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47545,
@@ -755,6 +774,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 41423244,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47531,
@@ -764,6 +784,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 32394133,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47530,
@@ -773,6 +794,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 43015519,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47520,
@@ -782,6 +804,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 42230604,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47496,
@@ -791,6 +814,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 41710968,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47486,
@@ -800,6 +824,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 42295944,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47486,
@@ -809,6 +834,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 41761321,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47460,
@@ -818,6 +844,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 30481198,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47439,
@@ -827,6 +854,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 41554041,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47395,
@@ -836,6 +864,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 42815907,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47394,
@@ -845,6 +874,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 41157112,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47372,
@@ -854,6 +884,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 31626792,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47369,
@@ -863,6 +894,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 41828969,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47348,
@@ -872,6 +904,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 40010773,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47339,
@@ -881,6 +914,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 43480570,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47325,
@@ -890,6 +924,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 41822475,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 47294,
@@ -899,6 +934,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 43018490,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 24276,
@@ -908,6 +944,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 23579440,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 4,
@@ -917,6 +954,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 28409,
+          checkedAt: 1706526408000,
         },
         {
           docsCount: 0,
@@ -926,6 +964,7 @@ describe('helpers', () => {
           pattern: 'auditbeat-*',
           patternDocsCount: 1118155,
           sizeInBytes: 247,
+          checkedAt: 1706526408000,
         },
       ],
       pageSize: 10,

diff --git a/...lution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts b/...lution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/pattern/helpers.ts
@@ -172,6 +172,7 @@ export const getSummaryTableItems = ({
     pattern,
     patternDocsCount,
     sizeInBytes: getSizeInBytes({ stats, indexName }),
+    checkedAt: results?.[indexName]?.checkedAt,
   }));
 
   return orderBy([sortByColumn], [sortByDirection], summaryTableItems);

diff --git a/...ata_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx b/...ata_quality_dashboard/impl/data_quality/data_quality_panel/summary_table/helpers.test.tsx
@@ -143,6 +143,7 @@ describe('helpers', () => {
       pattern: 'auditbeat-*',
       patternDocsCount: 57410,
       sizeInBytes: 103344068,
+      checkedAt: Date.now(),
     };
 
     const hasIncompatible: IndexSummaryTableItem = {
@@ -188,6 +189,7 @@ describe('helpers', () => {
         },
         { field: 'ilmPhase', name: 'ILM Phase', sortable: true, truncateText: false },
         { field: 'sizeInBytes', name: 'Size', sortable: true, truncateText: false },
+        { field: 'checkedAt', name: 'Last check', sortable: true, truncateText: false },
       ]);
     });