[Security Solution] Data Quality Dashboard persistence

.github/CODEOWNERS

@@ -352,9 +352,9 @@ packages/kbn-doc-links @elastic/docs
 packages/kbn-docs-utils @elastic/kibana-operations
 packages/kbn-dom-drag-drop @elastic/kibana-visualizations @elastic/kibana-data-discovery
 packages/kbn-ebt-tools @elastic/kibana-core
-packages/kbn-ecs @elastic/kibana-core @elastic/security-threat-hunting-investigations
-x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
-x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-investigations
+packages/kbn-ecs @elastic/kibana-core @elastic/security-threat-hunting-explore
+x-pack/packages/security-solution/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore
+x-pack/plugins/ecs_data_quality_dashboard @elastic/security-threat-hunting-explore
 packages/kbn-elastic-agent-utils @elastic/obs-ux-logs-team
 x-pack/packages/kbn-elastic-assistant @elastic/security-solution
 x-pack/packages/kbn-elastic-assistant-common @elastic/security-solution

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/data_quality_panel/index_properties/index.tsx

@@ -44,7 +44,7 @@ import { useAddToNewCase } from '../../use_add_to_new_case';
 import { useMappings } from '../../use_mappings';
 import { useUnallowedValues } from '../../use_unallowed_values';
 import { useDataQualityContext } from '../data_quality_context';
-import { getSizeInBytes, postResult } from '../../helpers';
+import { formatStorageResult, postStorageResult, getSizeInBytes } from '../../helpers';
 
 const EMPTY_MARKDOWN_COMMENTS: string[] = [];
 
@@ -268,7 +268,7 @@ const IndexPropertiesComponent: React.FC<Props> = ({
         updatePatternRollup(updatedRollup);
 
         if (indexId && requestTime != null && requestTime > 0 && partitionedFieldMetadata) {
-          const checkMetadata = {
+          const report = {
             batchId: uuidv4(),
             ecsVersion: EcsVersion,
             errorCount: error ? 1 : 0,
@@ -294,10 +294,13 @@ const IndexPropertiesComponent: React.FC<Props> = ({
               partitionedFieldMetadata.incompatible
             ),
           };
-          telemetryEvents.reportDataQualityIndexChecked?.(checkMetadata);
+          telemetryEvents.reportDataQualityIndexChecked?.(report);
 
-          const result = { meta: checkMetadata, rollup: updatedRollup };
-          postResult({ result, httpFetch, toasts, abortController: new AbortController() });
+          const result = updatedRollup.results[indexName];
+          if (result) {
+            const storageResult = formatStorageResult({ result, report, partitionedFieldMetadata });
+            postStorageResult({ storageResult, httpFetch, toasts });
+          }
         }
       }
     }

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.test.ts

@@ -35,9 +35,9 @@ import {
   getTotalSizeInBytes,
   hasValidTimestampMapping,
   isMappingCompatible,
-  postResult,
-  getResults,
-  ResultData,
+  postStorageResult,
+  getStorageResults,
+  StorageResult,
 } from './helpers';
 import {
   hostNameWithTextMapping,
@@ -1495,18 +1495,18 @@ describe('helpers', () => {
     });
   });
 
-  describe('postResult', () => {
+  describe('postStorageResult', () => {
     const { fetch } = httpServiceMock.createStartContract();
     const { toasts } = notificationServiceMock.createStartContract();
     beforeEach(() => {
       fetch.mockClear();
     });
 
     test('it posts the result', async () => {
-      const result = { meta: {}, rollup: {} } as unknown as ResultData;
-      await postResult({
+      const storageResult = { indexName: 'test' } as unknown as StorageResult;
+      await postStorageResult({
+        storageResult,
         httpFetch: fetch,
-        result,
         abortController: new AbortController(),
         toasts,
       });
@@ -1515,55 +1515,55 @@ describe('helpers', () => {
         '/internal/ecs_data_quality_dashboard/results',
         expect.objectContaining({
           method: 'POST',
-          body: JSON.stringify(result),
+          body: JSON.stringify(storageResult),
         })
       );
     });
 
     test('it throws error', async () => {
-      const result = { meta: {}, rollup: {} } as unknown as ResultData;
+      const storageResult = { indexName: 'test' } as unknown as StorageResult;
       fetch.mockRejectedValueOnce('test-error');
-      await postResult({
+      await postStorageResult({
         httpFetch: fetch,
-        result,
+        storageResult,
         abortController: new AbortController(),
         toasts,
       });
       expect(toasts.addError).toHaveBeenCalledWith('test-error', { title: expect.any(String) });
     });
   });
 
-  describe('getResults', () => {
+  describe('getStorageResults', () => {
     const { fetch } = httpServiceMock.createStartContract();
     const { toasts } = notificationServiceMock.createStartContract();
     beforeEach(() => {
       fetch.mockClear();
     });
 
     test('it gets the results', async () => {
-      await getResults({
+      await getStorageResults({
         httpFetch: fetch,
         abortController: new AbortController(),
-        patterns: ['auditbeat-*', 'packetbeat-*'],
+        pattern: 'auditbeat-*',
         toasts,
       });
 
       expect(fetch).toHaveBeenCalledWith(
         '/internal/ecs_data_quality_dashboard/results',
         expect.objectContaining({
           method: 'GET',
-          query: { patterns: 'auditbeat-*,packetbeat-*' },
+          query: { pattern: 'auditbeat-*' },
         })
       );
     });
 
     it('should catch error', async () => {
       fetch.mockRejectedValueOnce('test-error');
 
-      const results = await getResults({
+      const results = await getStorageResults({
         httpFetch: fetch,
         abortController: new AbortController(),
-        patterns: ['auditbeat-*', 'packetbeat-*'],
+        pattern: 'auditbeat-*',
         toasts,
       });
 

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/helpers.ts

@@ -23,6 +23,7 @@ import type {
   EcsMetadata,
   EnrichedFieldMetadata,
   ErrorSummary,
+  IlmPhase,
   PartitionedFieldMetadata,
   PartitionedFieldMetadataStats,
   PatternRollup,
@@ -449,51 +450,113 @@ export const getErrorSummaries = (
 
 export const RESULTS_API_ROUTE = '/internal/ecs_data_quality_dashboard/results';
 
-export interface ResultData {
-  meta: DataQualityIndexCheckedParams;
-  rollup: PatternRollup;
+export interface StorageResult {
+  batchId: string;
+  indexName: string;
+  isCheckAll: boolean;
+  docsCount: number;
+  totalFieldCount: number;
+  ecsFieldCount: number;
+  customFieldCount: number;
+  incompatibleFieldCount: number;
+  sameFamilyFieldCount: number;
+  sameFamilyFields: string[];
+  unallowedMappingFields: string[];
+  unallowedValueFields: string[];
+  sizeInBytes: number;
+  ilmPhase?: IlmPhase;
+  markdownComments: string[];
+  ecsVersion: string;
+  indexId: string;
+  error: string | null;
 }
 
-export async function postResult({
+export const formatStorageResult = ({
   result,
+  report,
+  partitionedFieldMetadata,
+}: {
+  result: DataQualityCheckResult;
+  report: DataQualityIndexCheckedParams;
+  partitionedFieldMetadata: PartitionedFieldMetadata;
+}): StorageResult => ({
+  batchId: report.batchId,
+  indexName: result.indexName,
+  isCheckAll: report.isCheckAll,
+  docsCount: result.docsCount ?? 0,
+  totalFieldCount: partitionedFieldMetadata.all.length,
+  ecsFieldCount: partitionedFieldMetadata.ecsCompliant.length,
+  customFieldCount: partitionedFieldMetadata.custom.length,
+  incompatibleFieldCount: partitionedFieldMetadata.incompatible.length,
+  sameFamilyFieldCount: partitionedFieldMetadata.sameFamily.length,
+  sameFamilyFields: report.sameFamilyFields ?? [],
+  unallowedMappingFields: report.unallowedMappingFields ?? [],
+  unallowedValueFields: report.unallowedValueFields ?? [],
+  sizeInBytes: report.sizeInBytes ?? 0,
+  ilmPhase: result.ilmPhase,
+  markdownComments: result.markdownComments,
+  ecsVersion: report.ecsVersion,
+  indexId: report.indexId,
+  error: result.error,
+});
+
+export const formatResultFromStorage = ({
+  storageResult,
+  pattern,
+}: {
+  storageResult: StorageResult;
+  pattern: string;
+}): DataQualityCheckResult => ({
+  docsCount: storageResult.docsCount,
+  error: storageResult.error,
+  ilmPhase: storageResult.ilmPhase,
+  incompatible: storageResult.incompatibleFieldCount,
+  indexName: storageResult.indexName,
+  markdownComments: storageResult.markdownComments,
+  sameFamily: storageResult.sameFamilyFieldCount,
+  pattern,
+});
+
+export async function postStorageResult({
+  storageResult,
   httpFetch,
   toasts,
-  abortController,
+  abortController = new AbortController(),
 }: {
-  result: ResultData;
+  storageResult: StorageResult;
   httpFetch: HttpHandler;
   toasts: IToasts;
-  abortController: AbortController;
+  abortController?: AbortController;
 }): Promise<void> {
   try {
     await httpFetch<void>(RESULTS_API_ROUTE, {
       method: 'POST',
       signal: abortController.signal,
       version: INTERNAL_API_VERSION,
-      body: JSON.stringify(result),
+      body: JSON.stringify(storageResult),
     });
   } catch (err) {
     toasts.addError(err, { title: i18n.POST_RESULT_ERROR_TITLE });
   }
 }
 
-export async function getResults({
-  patterns,
+export async function getStorageResults({
+  pattern,
   httpFetch,
   toasts,
   abortController,
 }: {
-  patterns: string[];
+  pattern: string;
   httpFetch: HttpHandler;
   toasts: IToasts;
   abortController: AbortController;
-}): Promise<ResultData[]> {
+}): Promise<StorageResult[]> {
   try {
-    const results = await httpFetch<ResultData[]>(RESULTS_API_ROUTE, {
+    const results = await httpFetch<StorageResult[]>(RESULTS_API_ROUTE, {
       method: 'GET',
       signal: abortController.signal,
       version: INTERNAL_API_VERSION,
-      query: { patterns: patterns.join(',') },
+      query: { pattern },
     });
     return results;
   } catch (err) {

x-pack/packages/security-solution/ecs_data_quality_dashboard/impl/data_quality/types.ts

@@ -186,8 +186,8 @@ export type DataQualityIndexCheckedParams = DataQualityCheckAllCompletedParams &
 
 export interface DataQualityCheckAllCompletedParams {
   batchId: string;
-  ecsVersion?: string;
-  isCheckAll?: boolean;
+  ecsVersion: string;
+  isCheckAll: boolean;
   numberOfDocuments?: number;
   numberOfIncompatibleFields?: number;
   numberOfIndices?: number;