[ML] Adds validation of field selected for log pattern analysis

x-pack/packages/ml/category_validator/common/types/categories.ts

@@ -79,3 +79,10 @@ export interface FieldExampleCheck {
    */
   message: string;
 }
+
+export interface FieldValidationResults {
+  examples?: CategoryFieldExample[];
+  sampleSize: number;
+  overallValidStatus: CATEGORY_EXAMPLES_VALIDATION_STATUS;
+  validationChecks: FieldExampleCheck[];
+}

x-pack/packages/ml/category_validator/index.ts

@@ -11,6 +11,7 @@ export type {
   CategoryFieldExample,
   FieldExampleCheck,
   Token,
+  FieldValidationResults,
 } from './common/types/categories';
 export {
   CATEGORY_EXAMPLES_ERROR_LIMIT,

x-pack/packages/ml/category_validator/src/examples.ts

@@ -210,7 +210,8 @@ export function categorizationExamplesProvider(client: IScopedClusterClient) {
     end: number,
     analyzer: CategorizationAnalyzer,
     runtimeMappings: RuntimeMappings | undefined,
-    indicesOptions: estypes.IndicesOptions | undefined
+    indicesOptions: estypes.IndicesOptions | undefined,
+    includeExamples = true
   ) {
     const resp = await categorizationExamples(
       indexPatternTitle,
@@ -229,6 +230,14 @@ export function categorizationExamplesProvider(client: IScopedClusterClient) {
     const sampleSize = examples.length;
     validationResults.createTokenCountResult(examples, sampleSize);
 
+    if (includeExamples === false) {
+      return {
+        overallValidStatus: validationResults.overallResult,
+        validationChecks: validationResults.results,
+        sampleSize,
+      };
+    }
+
     // sort examples by number of tokens, keeping track of their original order
     // with an origIndex property
     const sortedExamples = examples

x-pack/plugins/aiops/common/api/index.ts

@@ -15,6 +15,7 @@ import { streamReducer } from './stream_reducer';
 
 export const AIOPS_API_ENDPOINT = {
   LOG_RATE_ANALYSIS: '/internal/aiops/log_rate_analysis',
+  CATEGORIZATION_FIELD_VALIDATION: '/internal/aiops/categorization_field_validation',
 } as const;
 
 type AiopsApiEndpointKeys = keyof typeof AIOPS_API_ENDPOINT;

x-pack/plugins/aiops/common/api/log_categorization/schema.ts

@@ -6,6 +6,39 @@
  */
 
 import { schema, TypeOf } from '@kbn/config-schema';
+import { i18n } from '@kbn/i18n';
+import { isRuntimeField } from '@kbn/ml-runtime-field-utils';
+
+export const runtimeMappingsSchema = schema.object(
+  {},
+  {
+    unknowns: 'allow',
+    validate: (v: object) => {
+      if (Object.values(v).some((o) => !isRuntimeField(o))) {
+        return i18n.translate('xpack.aiops.invalidRuntimeFieldMessage', {
+          defaultMessage: 'Invalid runtime field',
+        });
+      }
+    },
+  }
+);
+
+export const indicesOptionsSchema = schema.object({
+  expand_wildcards: schema.maybe(
+    schema.arrayOf(
+      schema.oneOf([
+        schema.literal('all'),
+        schema.literal('open'),
+        schema.literal('closed'),
+        schema.literal('hidden'),
+        schema.literal('none'),
+      ])
+    )
+  ),
+  ignore_unavailable: schema.maybe(schema.boolean()),
+  allow_no_indices: schema.maybe(schema.boolean()),
+  ignore_throttled: schema.maybe(schema.boolean()),
+});
 
 export const categorizeSchema = schema.object({
   index: schema.string(),
@@ -18,3 +51,17 @@ export const categorizeSchema = schema.object({
 });
 
 export type CategorizeSchema = TypeOf<typeof categorizeSchema>;
+
+export const categorizationFieldValidationSchema = schema.object({
+  indexPatternTitle: schema.string(),
+  query: schema.any(),
+  size: schema.number(),
+  field: schema.string(),
+  timeField: schema.maybe(schema.string()),
+  start: schema.number(),
+  end: schema.number(),
+  analyzer: schema.maybe(schema.any()),
+  runtimeMappings: runtimeMappingsSchema,
+  indicesOptions: indicesOptionsSchema,
+  includeExamples: schema.boolean(),
+});

x-pack/plugins/aiops/public/application/utils/url_state.ts

@@ -8,7 +8,6 @@
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import type { Filter, Query } from '@kbn/es-query';
-import { isPopulatedObject } from '@kbn/ml-is-populated-object';
 
 import { SEARCH_QUERY_LANGUAGE, SearchQueryLanguage } from './search_utils';
 
@@ -40,6 +39,20 @@ export const getDefaultAiOpsListState = (
   ...overrides,
 });
 
-export const isFullAiOpsListState = (arg: unknown): arg is AiOpsFullIndexBasedAppState => {
-  return isPopulatedObject(arg, Object.keys(getDefaultAiOpsListState()));
+export interface LogCategorizationPageUrlState {
+  pageKey: 'logCategorization';
+  pageUrlState: LogCategorizationAppState;
+}
+
+export interface LogCategorizationAppState extends AiOpsFullIndexBasedAppState {
+  field: string | undefined;
+}
+
+export const getDefaultLogCategorizationAppState = (
+  overrides?: Partial<LogCategorizationAppState>
+): LogCategorizationAppState => {
+  return {
+    field: undefined,
+    ...getDefaultAiOpsListState(overrides),
+  };
 };

x-pack/plugins/aiops/public/components/log_categorization/category_table/category_table.tsx

@@ -24,7 +24,7 @@ import { Filter } from '@kbn/es-query';
 import { useDiscoverLinks, createFilter, QueryMode, QUERY_MODE } from '../use_discover_links';
 import { MiniHistogram } from '../../mini_histogram';
 import { useEuiTheme } from '../../../hooks/use_eui_theme';
-import type { AiOpsFullIndexBasedAppState } from '../../../application/utils/url_state';
+import type { LogCategorizationAppState } from '../../../application/utils/url_state';
 import type { EventRate, Category, SparkLinesPerCategory } from '../use_categorize_request';
 import { useTableState } from './use_table_state';
 import { getLabels } from './labels';
@@ -37,7 +37,7 @@ interface Props {
   dataViewId: string;
   selectedField: DataViewField | string | undefined;
   timefilter: TimefilterContract;
-  aiopsListState: AiOpsFullIndexBasedAppState;
+  aiopsListState: LogCategorizationAppState;
   pinnedCategory: Category | null;
   setPinnedCategory: (category: Category | null) => void;
   selectedCategory: Category | null;

x-pack/plugins/aiops/public/components/log_categorization/category_table/table_header.tsx

@@ -33,7 +33,7 @@ export const TableHeader: FC<Props> = ({
           <EuiText size="s" data-test-subj="aiopsLogPatternsFoundCount">
             <FormattedMessage
               id="xpack.aiops.logCategorization.counts"
-              defaultMessage="{count} patterns found"
+              defaultMessage="{count} {count, plural, one {pattern} other {patterns}} found"
               values={{ count: categoriesCount }}
             />
             {selectedCategoriesCount > 0 ? (

x-pack/plugins/aiops/public/components/log_categorization/category_validation_callout.tsx

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC } from 'react';
+import {
+  FieldValidationResults,
+  CATEGORY_EXAMPLES_VALIDATION_STATUS,
+} from '@kbn/ml-category-validator';
+
+import { EuiCallOut } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+
+interface Props {
+  validationResults: FieldValidationResults | null;
+}
+
+export const FieldValidationCallout: FC<Props> = ({ validationResults }) => {
+  if (validationResults === null) {
+    return null;
+  }
+
+  if (validationResults.overallValidStatus === CATEGORY_EXAMPLES_VALIDATION_STATUS.VALID) {
+    return null;
+  }
+
+  return (
+    <>
+      {validationResults !== null ? (
+        <EuiCallOut
+          color="warning"
+          title={i18n.translate('xpack.aiops.logCategorization.fieldValidationTitle', {
+            defaultMessage: 'The selected field is possibly not suitable for pattern analysis',
+          })}
+        >
+          {validationResults.validationChecks
+            .filter((check) => check.valid !== CATEGORY_EXAMPLES_VALIDATION_STATUS.VALID)
+            .map((check) => (
+              <div key={check.id}>{check.message}</div>
+            ))}
+        </EuiCallOut>
+      ) : null}
+    </>
+  );
+};

x-pack/plugins/aiops/public/components/log_categorization/information_text.tsx

@@ -57,7 +57,7 @@ export const InformationText: FC<Props> = ({
             <h2>
               <FormattedMessage
                 id="xpack.aiops.logCategorization.emptyPromptTitle"
-                defaultMessage="Select a text field and click run categorization to start analysis"
+                defaultMessage="Select a text field and click run pattern analysis to start analysis"
               />
             </h2>
           }

x-pack/plugins/aiops/public/components/log_categorization/log_categorization_for_flyout.tsx

@@ -21,6 +21,7 @@ import {
 import { buildEmptyFilter, Filter } from '@kbn/es-query';
 
 import { usePageUrlState } from '@kbn/ml-url-state';
+import type { FieldValidationResults } from '@kbn/ml-category-validator';
 import { useData } from '../../hooks/use_data';
 import { useSearch } from '../../hooks/use_search';
 import { useCategorizeRequest } from './use_categorize_request';
@@ -32,11 +33,12 @@ import { createMergedEsQuery } from '../../application/utils/search_utils';
 import { SamplingMenu } from './sampling_menu';
 import { TechnicalPreviewBadge } from './technical_preview_badge';
 import { LoadingCategorization } from './loading_categorization';
+import { useValidateFieldRequest } from './use_validate_category_field';
 import {
-  type AiOpsPageUrlState,
-  getDefaultAiOpsListState,
-  isFullAiOpsListState,
+  type LogCategorizationPageUrlState,
+  getDefaultLogCategorizationAppState,
 } from '../../application/utils/url_state';
+import { FieldValidationCallout } from './category_validation_callout';
 
 export interface LogCategorizationPageProps {
   dataView: DataView;
@@ -60,14 +62,21 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
     },
     uiSettings,
   } = useAiopsAppContext();
+
+  const { runValidateFieldRequest, cancelRequest: cancelValidationRequest } =
+    useValidateFieldRequest();
   const { euiTheme } = useEuiTheme();
   const { filters, query } = useMemo(() => getState(), [getState]);
 
   const mounted = useRef(false);
-  const { runCategorizeRequest, cancelRequest, randomSampler } = useCategorizeRequest();
-  const [aiopsListState] = usePageUrlState<AiOpsPageUrlState>(
-    'AIOPS_INDEX_VIEWER',
-    getDefaultAiOpsListState({
+  const {
+    runCategorizeRequest,
+    cancelRequest: cancelCategorizationRequest,
+    randomSampler,
+  } = useCategorizeRequest();
+  const [stateFromUrl] = usePageUrlState<LogCategorizationPageUrlState>(
+    'logCategorization',
+    getDefaultLogCategorizationAppState({
       searchQuery: createMergedEsQuery(query, filters, dataView, uiSettings),
     })
   );
@@ -80,6 +89,14 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
     categories: Category[];
     sparkLines: SparkLinesPerCategory;
   } | null>(null);
+  const [fieldValidationResult, setFieldValidationResult] = useState<FieldValidationResults | null>(
+    null
+  );
+
+  const cancelRequest = useCallback(() => {
+    cancelValidationRequest();
+    cancelCategorizationRequest();
+  }, [cancelCategorizationRequest, cancelValidationRequest]);
 
   useEffect(
     function cancelRequestOnLeave() {
@@ -94,7 +111,7 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
 
   const { searchQueryLanguage, searchString, searchQuery } = useSearch(
     { dataView, savedSearch: selectedSavedSearch },
-    aiopsListState,
+    stateFromUrl,
     true
   );
 
@@ -109,7 +126,8 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
   );
 
   const loadCategories = useCallback(async () => {
-    const { title: index, timeFieldName: timeField } = dataView;
+    const { getIndexPattern, timeFieldName: timeField } = dataView;
+    const index = getIndexPattern();
 
     if (selectedField === undefined || timeField === undefined) {
       return;
@@ -119,20 +137,35 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
 
     setLoading(true);
     setData(null);
+    setFieldValidationResult(null);
 
     try {
-      const { categories, sparkLinesPerCategory: sparkLines } = await runCategorizeRequest(
-        index,
-        selectedField.name,
-        timeField,
-        earliest,
-        latest,
-        searchQuery,
-        intervalMs
-      );
+      const [validationResult, categorizationResult] = await Promise.all([
+        runValidateFieldRequest(
+          index,
+          selectedField.name,
+          timeField,
+          earliest,
+          latest,
+          searchQuery
+        ),
+        runCategorizeRequest(
+          index,
+          selectedField.name,
+          timeField,
+          earliest,
+          latest,
+          searchQuery,
+          intervalMs
+        ),
+      ]);
 
       if (mounted.current === true) {
-        setData({ categories, sparkLines });
+        setFieldValidationResult(validationResult);
+        setData({
+          categories: categorizationResult.categories,
+          sparkLines: categorizationResult.sparkLinesPerCategory,
+        });
       }
     } catch (error) {
       toasts.addError(error, {
@@ -149,10 +182,11 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
     dataView,
     selectedField,
     cancelRequest,
-    runCategorizeRequest,
+    runValidateFieldRequest,
     earliest,
     latest,
     searchQuery,
+    runCategorizeRequest,
     intervalMs,
     toasts,
   ]);
@@ -217,6 +251,8 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
         </EuiFlexGroup>
       </EuiFlyoutHeader>
       <EuiFlyoutBody data-test-subj="mlJobSelectorFlyoutBody">
+        <FieldValidationCallout validationResults={fieldValidationResult} />
+
         {loading === true ? <LoadingCategorization onClose={onClose} /> : null}
 
         <InformationText
@@ -226,13 +262,10 @@ export const LogCategorizationFlyout: FC<LogCategorizationPageProps> = ({
           fieldSelected={selectedField !== null}
         />
 
-        {loading === false &&
-        data !== null &&
-        data.categories.length > 0 &&
-        isFullAiOpsListState(aiopsListState) ? (
+        {loading === false && data !== null && data.categories.length > 0 ? (
           <CategoryTable
             categories={data.categories}
-            aiopsListState={aiopsListState}
+            aiopsListState={stateFromUrl}
             dataViewId={dataView.id!}
             eventRate={eventRate}
             sparkLines={data.sparkLines}