[ML] Log pattern analysis field validation (elastic#162319)

Uses the recently created [category validation
package](elastic#161261) to perform
validation on the field selected for pattern analysis.

If the field is considered unsuitable for categorization, a warning
callout is displayed which lists the reasons it is unsuitable.
If the field is suitable, no callout is displayed.

Other changes:
- Adds the selected field to the URL state, so it is remembered on page
refresh.
- If no field is in the URL, it will look for a field called `message`
in the data view and auto select it.
- renames the ML route `/jobs/categorization_field_examples` to
`/jobs/categorization_field_validation` as it is a more accurate name
and it's consistent with the newly added route in AIOPs.

**Log Pattern Analysis page in ML**


![image](https://github.com/elastic/kibana/assets/22172091/c0dfda8b-bc34-48b7-9e71-8bae9e65bdf3)


**Log Pattern Analysis flyout in Discover**


![image](https://github.com/elastic/kibana/assets/22172091/b4d251f3-bae6-424f-9891-bda57ba1673d)

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

x-pack/packages/ml/category_validator/common/types/categories.ts

@@ -79,3 +79,32 @@ export interface FieldExampleCheck {
    */
   message: string;
 }
+
+/**
+ * Validation results for a specific field.
+ */
+export interface FieldValidationResults {
+  /**
+   * An array of example objects representing category field examples.
+   * @type {CategoryFieldExample[]}
+   */
+  examples?: CategoryFieldExample[];
+
+  /**
+   * The total number of examples used for validation.
+   * @type {number}
+   */
+  sampleSize: number;
+
+  /**
+   * The overall validation status of the category examples.
+   * @type {CATEGORY_EXAMPLES_VALIDATION_STATUS}
+   */
+  overallValidStatus: CATEGORY_EXAMPLES_VALIDATION_STATUS;
+
+  /**
+   * An array of validation checks performed on each example.
+   * @type {FieldExampleCheck[]}
+   */
+  validationChecks: FieldExampleCheck[];
+}

x-pack/packages/ml/category_validator/index.ts

@@ -11,6 +11,7 @@ export type {
   CategoryFieldExample,
   FieldExampleCheck,
   Token,
+  FieldValidationResults,
 } from './common/types/categories';
 export {
   CATEGORY_EXAMPLES_ERROR_LIMIT,

x-pack/packages/ml/category_validator/src/examples.ts

@@ -210,7 +210,8 @@ export function categorizationExamplesProvider(client: IScopedClusterClient) {
     end: number,
     analyzer: CategorizationAnalyzer,
     runtimeMappings: RuntimeMappings | undefined,
-    indicesOptions: estypes.IndicesOptions | undefined
+    indicesOptions: estypes.IndicesOptions | undefined,
+    includeExamples = true
   ) {
     const resp = await categorizationExamples(
       indexPatternTitle,
@@ -229,6 +230,14 @@ export function categorizationExamplesProvider(client: IScopedClusterClient) {
     const sampleSize = examples.length;
     validationResults.createTokenCountResult(examples, sampleSize);
 
+    if (includeExamples === false) {
+      return {
+        overallValidStatus: validationResults.overallResult,
+        validationChecks: validationResults.results,
+        sampleSize,
+      };
+    }
+
     // sort examples by number of tokens, keeping track of their original order
     // with an origIndex property
     const sortedExamples = examples

x-pack/plugins/aiops/common/api/index.ts

@@ -15,6 +15,7 @@ import { streamReducer } from './stream_reducer';
 
 export const AIOPS_API_ENDPOINT = {
   LOG_RATE_ANALYSIS: '/internal/aiops/log_rate_analysis',
+  CATEGORIZATION_FIELD_VALIDATION: '/internal/aiops/categorization_field_validation',
 } as const;
 
 type AiopsApiEndpointKeys = keyof typeof AIOPS_API_ENDPOINT;

x-pack/plugins/aiops/common/api/log_categorization/schema.ts

@@ -6,6 +6,39 @@
  */
 
 import { schema, TypeOf } from '@kbn/config-schema';
+import { i18n } from '@kbn/i18n';
+import { isRuntimeField } from '@kbn/ml-runtime-field-utils';
+
+export const runtimeMappingsSchema = schema.object(
+  {},
+  {
+    unknowns: 'allow',
+    validate: (v: object) => {
+      if (Object.values(v).some((o) => !isRuntimeField(o))) {
+        return i18n.translate('xpack.aiops.invalidRuntimeFieldMessage', {
+          defaultMessage: 'Invalid runtime field',
+        });
+      }
+    },
+  }
+);
+
+export const indicesOptionsSchema = schema.object({
+  expand_wildcards: schema.maybe(
+    schema.arrayOf(
+      schema.oneOf([
+        schema.literal('all'),
+        schema.literal('open'),
+        schema.literal('closed'),
+        schema.literal('hidden'),
+        schema.literal('none'),
+      ])
+    )
+  ),
+  ignore_unavailable: schema.maybe(schema.boolean()),
+  allow_no_indices: schema.maybe(schema.boolean()),
+  ignore_throttled: schema.maybe(schema.boolean()),
+});
 
 export const categorizeSchema = schema.object({
   index: schema.string(),
@@ -18,3 +51,17 @@ export const categorizeSchema = schema.object({
 });
 
 export type CategorizeSchema = TypeOf<typeof categorizeSchema>;
+
+export const categorizationFieldValidationSchema = schema.object({
+  indexPatternTitle: schema.string(),
+  query: schema.any(),
+  size: schema.number(),
+  field: schema.string(),
+  timeField: schema.maybe(schema.string()),
+  start: schema.number(),
+  end: schema.number(),
+  analyzer: schema.maybe(schema.any()),
+  runtimeMappings: runtimeMappingsSchema,
+  indicesOptions: indicesOptionsSchema,
+  includeExamples: schema.boolean(),
+});

x-pack/plugins/aiops/public/application/utils/url_state.ts

@@ -8,7 +8,6 @@
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { Filter, Query } from '@kbn/es-query';
-import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 
 import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from './search_utils';
 
@@ -40,6 +39,20 @@ export const getDefaultAiOpsListState = (
   ...overrides,
 });
 
-export const isFullAiOpsListState = (arg: unknown): arg is AiOpsFullIndexBasedAppState => {
-  return isPopulatedObject(arg, Object.keys(getDefaultAiOpsListState()));
+export interface LogCategorizationPageUrlState {
+  pageKey: 'logCategorization';
+  pageUrlState: LogCategorizationAppState;
+}
+
+export interface LogCategorizationAppState extends AiOpsFullIndexBasedAppState {
+  field: string | undefined;
+}
+
+export const getDefaultLogCategorizationAppState = (
+  overrides?: Partial<LogCategorizationAppState>
+): LogCategorizationAppState => {
+  return {
+    field: undefined,
+    ...getDefaultAiOpsListState(overrides),
+  };
 };

x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx

@@ -24,7 +24,7 @@ import { Filter } from '@kbn/es-query';
 import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
 import { MiniHistogram } from '../../mini_histogram';
 import { useEuiTheme } from '../../../hooks/use_eui_theme';
-import type { AiOpsFullIndexBasedAppState } from '../../../application/utils/url_state';
+import type { LogCategorizationAppState } from '../../../application/utils/url_state';
 import type { EventRate, Category, SparkLinesPerCategory } from '../use_categorize_request';
 import { useTableState } from './use_table_state';
 import { getLabels } from './labels';
@@ -37,7 +37,7 @@ interface Props {
   dataViewId: string;
   selectedField: DataViewField | string | undefined;
   timefilter: TimefilterContract;
-  aiopsListState: AiOpsFullIndexBasedAppState;
+  aiopsListState: LogCategorizationAppState;
   pinnedCategory: Category | null;
   setPinnedCategory: (category: Category | null) => void;
   selectedCategory: Category | null;

x-pack/plugins/aiops/public/components/log_categorization/category_table/table_header.tsx

@@ -33,7 +33,7 @@ export const TableHeader: FC<Props> = ({
           <EuiText size="s" data-test-subj="aiopsLogPatternsFoundCount">
             <FormattedMessage
               id="xpack.aiops.logCategorization.counts"
-              defaultMessage="{count} patterns found"
+              defaultMessage="{count} {count, plural, one {pattern} other {patterns}} found"
               values={{ count: categoriesCount }}
             />
             {selectedCategoriesCount > 0 ? (

x-pack/plugins/aiops/public/components/log_categorization/category_validation_callout.tsx

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC } from 'react';
+import {
+  FieldValidationResults,
+  CATEGORY_EXAMPLES_VALIDATION_STATUS,
+} from '@kbn/ml-category-validator';
+
+import { EuiCallOut } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+interface Props {
+  validationResults: FieldValidationResults | null;
+}
+
+export const FieldValidationCallout: FC<Props> = ({ validationResults }) => {
+  if (validationResults === null) {
+    return null;
+  }
+
+  if (validationResults.overallValidStatus === CATEGORY_EXAMPLES_VALIDATION_STATUS.VALID) {
+    return null;
+  }
+
+  return (
+    <EuiCallOut
+      color="warning"
+      title={i18n.translate('xpack.aiops.logCategorization.fieldValidationTitle', {
+        defaultMessage: 'The selected field is possibly not suitable for pattern analysis',
+      })}
+    >
+      {validationResults.validationChecks
+        .filter((check) => check.valid !== CATEGORY_EXAMPLES_VALIDATION_STATUS.VALID)
+        .map((check) => (
+          <div key={check.id}>{check.message}</div>
+        ))}
+    </EuiCallOut>
+  );
+};

x-pack/plugins/aiops/public/components/log_categorization/information_text.tsx

@@ -57,7 +57,7 @@ export const InformationText: FC<Props> = ({
             <h2>
               <FormattedMessage
                 id="xpack.aiops.logCategorization.emptyPromptTitle"
-                defaultMessage="Select a text field and click run categorization to start analysis"
+                defaultMessage="Select a text field and click run pattern analysis to start analysis"
               />
             </h2>
           }