[ML] Explain Log Rate Spikes: Additional unit tests. (#147451)

This breaks out inline code in
`x-pack/plugins/aiops/server/routes/explain_log_rate_spikes.ts` to
functions in separate files and adds jest unit tests for each function.
The mocks used for jest unit tests are used as expected data in the API
integration tests. This allows to make sure that the static mocks are
still up to date should we have to update API integration tests based on
upstream tests.

x-pack/plugins/aiops/common/__mocks__/artificial_logs/change_point_groups.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ChangePointGroup } from '@kbn/ml-agg-utils';
+
+export const changePointGroups: ChangePointGroup[] = [
+  {
+    id: '2038579476',
+    group: [
+      { fieldName: 'response_code', fieldValue: '500' },
+      { fieldName: 'url', fieldValue: 'home.php' },
+    ],
+    docCount: 792,
+    pValue: 0.010770456205312423,
+  },
+];

x-pack/plugins/aiops/common/__mocks__/artificial_logs/change_points.ts

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const changePoints = [
+  {
+    fieldName: 'response_code',
+    fieldValue: '500',
+    doc_count: 1821,
+    bg_count: 553,
+    total_doc_count: 4671,
+    total_bg_count: 1975,
+    score: 26.546201745993947,
+    pValue: 2.9589053032077285e-12,
+    normalizedScore: 0.7814127409489161,
+  },
+  {
+    fieldName: 'url',
+    fieldValue: 'home.php',
+    doc_count: 1742,
+    bg_count: 632,
+    total_doc_count: 4671,
+    total_bg_count: 1975,
+    score: 4.53094842981472,
+    pValue: 0.010770456205312423,
+    normalizedScore: 0.10333028878375965,
+  },
+  {
+    fieldName: 'url',
+    fieldValue: 'login.php',
+    doc_count: 1742,
+    bg_count: 632,
+    total_doc_count: 4671,
+    total_bg_count: 1975,
+    score: 4.53094842981472,
+    pValue: 0.010770456205312423,
+    normalizedScore: 0.10333028878375965,
+  },
+  {
+    fieldName: 'user',
+    fieldValue: 'Peter',
+    doc_count: 1981,
+    bg_count: 553,
+    total_doc_count: 4671,
+    total_bg_count: 1975,
+    score: 47.34435085428873,
+    pValue: 2.7454255728359757e-21,
+    normalizedScore: 0.8327337555873047,
+  },
+];

x-pack/plugins/aiops/common/__mocks__/artificial_logs/fields.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export const fields = ['response_code', 'url', 'user'];

x-pack/plugins/aiops/common/__mocks__/artificial_logs/filtered_frequent_items.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ItemsetResult } from '../../types';
+
+export const filteredFrequentItems: ItemsetResult[] = [
+  {
+    set: { response_code: '500', url: 'home.php' },
+    size: 2,
+    maxPValue: 0.010770456205312423,
+    doc_count: 792,
+    support: 0.5262458471760797,
+    total_doc_count: 1505,
+  },
+  {
+    set: { user: 'Peter', url: 'home.php' },
+    size: 2,
+    maxPValue: 0.010770456205312423,
+    doc_count: 634,
+    support: 0.4212624584717608,
+    total_doc_count: 1505,
+  },
+];

x-pack/plugins/aiops/common/__mocks__/artificial_logs/final_change_point_groups.ts

@@ -0,0 +1,27 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ChangePointGroup } from '@kbn/ml-agg-utils';
+
+export const finalChangePointGroups: ChangePointGroup[] = [
+  {
+    id: '2038579476',
+    group: [
+      { fieldName: 'response_code', fieldValue: '500', duplicate: false },
+      { fieldName: 'url', fieldValue: 'home.php', duplicate: false },
+      { fieldName: 'url', fieldValue: 'login.php', duplicate: false },
+    ],
+    docCount: 792,
+    pValue: 0.010770456205312423,
+  },
+  {
+    id: '817080373',
+    group: [{ fieldName: 'user', fieldValue: 'Peter', duplicate: false }],
+    docCount: 1981,
+    pValue: 2.7454255728359757e-21,
+  },
+];

x-pack/plugins/aiops/common/__mocks__/artificial_logs/frequent_items.ts

@@ -0,0 +1,59 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ItemsetResult } from '../../types';
+
+export const frequentItems: ItemsetResult[] = [
+  {
+    set: { response_code: '500', url: 'home.php' },
+    size: 2,
+    maxPValue: 0.010770456205312423,
+    doc_count: 792,
+    support: 0.5262458471760797,
+    total_doc_count: 1505,
+  },
+  {
+    set: { user: 'Peter', url: 'home.php' },
+    size: 2,
+    maxPValue: 0.010770456205312423,
+    doc_count: 634,
+    support: 0.4212624584717608,
+    total_doc_count: 1505,
+  },
+  {
+    set: { response_code: '500', user: 'Mary', url: 'home.php' },
+    size: 3,
+    maxPValue: 0.010770456205312423,
+    doc_count: 396,
+    support: 0.26312292358803985,
+    total_doc_count: 1505,
+  },
+  {
+    set: { response_code: '500', user: 'Paul', url: 'home.php' },
+    size: 3,
+    maxPValue: 0.010770456205312423,
+    doc_count: 396,
+    support: 0.26312292358803985,
+    total_doc_count: 1505,
+  },
+  {
+    set: { response_code: '404', user: 'Peter', url: 'home.php' },
+    size: 3,
+    maxPValue: 0.010770456205312423,
+    doc_count: 317,
+    support: 0.2106312292358804,
+    total_doc_count: 1505,
+  },
+  {
+    set: { response_code: '200', user: 'Peter', url: 'home.php' },
+    size: 3,
+    maxPValue: 0.010770456205312423,
+    doc_count: 317,
+    support: 0.2106312292358804,
+    total_doc_count: 1505,
+  },
+];

x-pack/plugins/aiops/common/__mocks__/farequote/change_point_groups.ts

@@ -0,0 +1,41 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ChangePointGroup } from '@kbn/ml-agg-utils';
+
+export const changePointGroups: ChangePointGroup[] = [
+  {
+    id: 'group-1',
+    group: [
+      {
+        fieldName: 'custom_field.keyword',
+        fieldValue: 'deviation',
+      },
+      {
+        fieldName: 'airline',
+        fieldValue: 'UAL',
+      },
+    ],
+    docCount: 101,
+    pValue: 0.01,
+  },
+  {
+    id: 'group-2',
+    group: [
+      {
+        fieldName: 'custom_field.keyword',
+        fieldValue: 'deviation',
+      },
+      {
+        fieldName: 'airline',
+        fieldValue: 'AAL',
+      },
+    ],
+    docCount: 49,
+    pValue: 0.001,
+  },
+];

x-pack/plugins/aiops/common/types.ts

@@ -0,0 +1,33 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ChangePoint, FieldValuePair } from '@kbn/ml-agg-utils';
+
+export interface ChangePointDuplicateGroup {
+  keys: Pick<ChangePoint, keyof ChangePoint>;
+  group: ChangePoint[];
+}
+
+export type FieldValuePairCounts = Record<string, Record<string, number>>;
+
+export interface ItemsetResult {
+  set: Record<FieldValuePair['fieldName'], FieldValuePair['fieldValue']>;
+  size: number;
+  maxPValue: number;
+  doc_count: number;
+  support: number;
+  total_doc_count: number;
+}
+
+export interface SimpleHierarchicalTreeNode {
+  name: string;
+  set: FieldValuePair[];
+  docCount: number;
+  pValue: number | null;
+  children: SimpleHierarchicalTreeNode[];
+  addNode: (node: SimpleHierarchicalTreeNode) => void;
+}

x-pack/plugins/aiops/public/application/utils/query_utils.test.ts

@@ -7,7 +7,7 @@
 
 import type { ChangePoint } from '@kbn/ml-agg-utils';
 
-import type { GroupTableItem } from '../../components/spike_analysis_table/spike_analysis_table_groups';
+import type { GroupTableItem } from '../../components/spike_analysis_table/types';
 
 import { buildBaseFilterCriteria } from './query_utils';
 

x-pack/plugins/aiops/public/application/utils/query_utils.ts

@@ -16,7 +16,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { Query } from '@kbn/es-query';
 import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 import type { ChangePoint, FieldValuePair } from '@kbn/ml-agg-utils';
-import type { GroupTableItem } from '../../components/spike_analysis_table/spike_analysis_table_groups';
+import type { GroupTableItem } from '../../components/spike_analysis_table/types';
 
 /*
  * Contains utility functions for building and processing queries.

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_analysis.tsx

@@ -25,14 +25,17 @@ import type { WindowParameters } from '@kbn/aiops-utils';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n-react';
 import type { Query } from '@kbn/es-query';
-import type { FieldValuePair } from '@kbn/ml-agg-utils';
 
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
 import { initialState, streamReducer } from '../../../common/api/stream_reducer';
 import type { ApiExplainLogRateSpikes } from '../../../common/api';
 
-import { SpikeAnalysisGroupsTable } from '../spike_analysis_table';
-import { SpikeAnalysisTable } from '../spike_analysis_table';
+import {
+  getGroupTableItems,
+  SpikeAnalysisTable,
+  SpikeAnalysisGroupsTable,
+} from '../spike_analysis_table';
+import {} from '../spike_analysis_table';
 import { useSpikeAnalysisTableRowContext } from '../spike_analysis_table/spike_analysis_table_row_provider';
 
 const groupResultsMessage = i18n.translate(
@@ -159,35 +162,10 @@ export const ExplainLogRateSpikesAnalysis: FC<ExplainLogRateSpikesAnalysisProps>
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
-  const groupTableItems = useMemo(() => {
-    const tableItems = data.changePointsGroups.map(({ id, group, docCount, histogram, pValue }) => {
-      const sortedGroup = group.sort((a, b) =>
-        a.fieldName > b.fieldName ? 1 : b.fieldName > a.fieldName ? -1 : 0
-      );
-      const dedupedGroup: FieldValuePair[] = [];
-      const repeatedValues: FieldValuePair[] = [];
-
-      sortedGroup.forEach((pair) => {
-        const { fieldName, fieldValue } = pair;
-        if (pair.duplicate === false) {
-          dedupedGroup.push({ fieldName, fieldValue });
-        } else {
-          repeatedValues.push({ fieldName, fieldValue });
-        }
-      });
-
-      return {
-        id,
-        docCount,
-        pValue,
-        group: dedupedGroup,
-        repeatedValues,
-        histogram,
-      };
-    });
-
-    return tableItems;
-  }, [data.changePointsGroups]);
+  const groupTableItems = useMemo(
+    () => getGroupTableItems(data.changePointsGroups),
+    [data.changePointsGroups]
+  );
 
   const shouldRerunAnalysis = useMemo(
     () =>

x-pack/plugins/aiops/public/components/explain_log_rate_spikes/explain_log_rate_spikes_page.tsx

@@ -38,7 +38,7 @@ import { SearchPanel } from '../search_panel';
 
 import { restorableDefaults } from './explain_log_rate_spikes_app_state';
 import { ExplainLogRateSpikesAnalysis } from './explain_log_rate_spikes_analysis';
-import type { GroupTableItem } from '../spike_analysis_table/spike_analysis_table_groups';
+import type { GroupTableItem } from '../spike_analysis_table/types';
 import { useSpikeAnalysisTableRowContext } from '../spike_analysis_table/spike_analysis_table_row_provider';
 
 // TODO port to `@emotion/react` once `useEuiBreakpoint` is available https://github.com/elastic/eui/pull/6057