diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fb12f72abd2f19..0d246e1992d247 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -484,6 +484,7 @@ x-pack/plugins/maps @elastic/kibana-gis x-pack/packages/maps/vector_tile_utils @elastic/kibana-gis x-pack/packages/ml/agg_utils @elastic/ml-ui x-pack/packages/ml/anomaly_utils @elastic/ml-ui +x-pack/packages/ml/category_validator @elastic/ml-ui x-pack/packages/ml/data_frame_analytics_utils @elastic/ml-ui x-pack/packages/ml/data_grid @elastic/ml-ui x-pack/packages/ml/date_picker @elastic/ml-ui diff --git a/package.json b/package.json index f515230d043660..6a3f61d15abe04 100644 --- a/package.json +++ b/package.json @@ -500,6 +500,7 @@ "@kbn/maps-vector-tile-utils": "link:x-pack/packages/maps/vector_tile_utils", "@kbn/ml-agg-utils": "link:x-pack/packages/ml/agg_utils", "@kbn/ml-anomaly-utils": "link:x-pack/packages/ml/anomaly_utils", + "@kbn/ml-category-validator": "link:x-pack/packages/ml/category_validator", "@kbn/ml-data-frame-analytics-utils": "link:x-pack/packages/ml/data_frame_analytics_utils", "@kbn/ml-data-grid": "link:x-pack/packages/ml/data_grid", "@kbn/ml-date-picker": "link:x-pack/packages/ml/date_picker", diff --git a/tsconfig.base.json b/tsconfig.base.json index c114d34233dc1b..d184f337e52480 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -962,6 +962,8 @@ "@kbn/ml-agg-utils/*": ["x-pack/packages/ml/agg_utils/*"], "@kbn/ml-anomaly-utils": ["x-pack/packages/ml/anomaly_utils"], "@kbn/ml-anomaly-utils/*": ["x-pack/packages/ml/anomaly_utils/*"], + "@kbn/ml-category-validator": ["x-pack/packages/ml/category_validator"], + "@kbn/ml-category-validator/*": ["x-pack/packages/ml/category_validator/*"], "@kbn/ml-data-frame-analytics-utils": ["x-pack/packages/ml/data_frame_analytics_utils"], "@kbn/ml-data-frame-analytics-utils/*": ["x-pack/packages/ml/data_frame_analytics_utils/*"], "@kbn/ml-data-grid": ["x-pack/packages/ml/data_grid"], diff --git a/x-pack/.i18nrc.json b/x-pack/.i18nrc.json index 6fbc0f935b9400..31d72d2aaab3ea 100644 --- a/x-pack/.i18nrc.json +++ b/x-pack/.i18nrc.json @@ -54,6 +54,7 @@ "packages/ml/data_grid", "packages/ml/date_picker", "packages/ml/trained_models_utils", + "packages/ml/category_validator", "plugins/ml" ], "xpack.monitoring": ["plugins/monitoring"], diff --git a/x-pack/packages/ml/category_validator/README.md b/x-pack/packages/ml/category_validator/README.md new file mode 100644 index 00000000000000..712f260721601a --- /dev/null +++ b/x-pack/packages/ml/category_validator/README.md @@ -0,0 +1,3 @@ +# @kbn/ml-category-validator + +Provides functions for validating data to see whether it is suitable for categorization diff --git a/x-pack/plugins/ml/common/constants/categorization_job.ts b/x-pack/packages/ml/category_validator/common/constants/categorization.ts similarity index 55% rename from x-pack/plugins/ml/common/constants/categorization_job.ts rename to x-pack/packages/ml/category_validator/common/constants/categorization.ts index 7436f884071a05..4bd5ab6710f9f5 100644 --- a/x-pack/plugins/ml/common/constants/categorization_job.ts +++ b/x-pack/packages/ml/category_validator/common/constants/categorization.ts @@ -6,76 +6,136 @@ */ import { i18n } from '@kbn/i18n'; -import { VALIDATION_RESULT } from '../types/categories'; -export const NUMBER_OF_CATEGORY_EXAMPLES = 5; +/** + * The number of category examples to use for analysis. + */ export const CATEGORY_EXAMPLES_SAMPLE_SIZE = 1000; + +/** + * The warning limit for category examples. If the category examples validation falls below this limit, a warning is triggered. + */ export const CATEGORY_EXAMPLES_WARNING_LIMIT = 0.75; + +/** + * The error limit for category examples. If the category examples validation falls below this limit, an error is triggered. + */ export const CATEGORY_EXAMPLES_ERROR_LIMIT = 0.02; +/** + * The valid token count for category examples. + */ export const VALID_TOKEN_COUNT = 3; + +/** + * The limit for the median line length of category examples. + */ export const MEDIAN_LINE_LENGTH_LIMIT = 400; + +/** + * The limit for the percentage of null values in category examples. + */ export const NULL_COUNT_PERCENT_LIMIT = 0.75; +/** + * Enum representing the validation status of category examples. + */ export enum CATEGORY_EXAMPLES_VALIDATION_STATUS { VALID = 'valid', PARTIALLY_VALID = 'partially_valid', INVALID = 'invalid', } +/** + * Enum representing the validation results for field examples. + */ +export enum VALIDATION_RESULT { + NO_EXAMPLES, + FAILED_TO_TOKENIZE, + TOO_MANY_TOKENS, + TOKEN_COUNT, + MEDIAN_LINE_LENGTH, + NULL_VALUES, + INSUFFICIENT_PRIVILEGES, +} + +/** + * Description for each validation result. + */ export const VALIDATION_CHECK_DESCRIPTION = { + /** + * Examples were successfully loaded. + */ [VALIDATION_RESULT.NO_EXAMPLES]: i18n.translate( 'xpack.ml.models.jobService.categorization.messages.validNoDataFound', { - defaultMessage: 'Examples were successfully loaded.', + defaultMessage: 'Examples were successfully loaded.', } - ), + ) as string, + /** + * The loaded examples were tokenized successfully. + */ [VALIDATION_RESULT.FAILED_TO_TOKENIZE]: i18n.translate( 'xpack.ml.models.jobService.categorization.messages.validFailureToGetTokens', { - defaultMessage: 'The examples loaded were tokenized successfully.', + defaultMessage: 'The loaded examples were tokenized successfully.', } - ), + ) as string, + /** + * More than {tokenCount} tokens per example were found in over {percentage}% of the loaded examples. + */ [VALIDATION_RESULT.TOKEN_COUNT]: i18n.translate( 'xpack.ml.models.jobService.categorization.messages.validTokenLength', { defaultMessage: - 'More than {tokenCount} tokens per example were found in over {percentage}% of the examples loaded.', + 'More than {tokenCount} tokens per example were found in over {percentage}% of the loaded examples.', values: { percentage: Math.floor(CATEGORY_EXAMPLES_WARNING_LIMIT * 100), tokenCount: VALID_TOKEN_COUNT, }, } - ), + ) as string, + /** + * The median line length of the loaded examples was less than {medianCharCount} characters. + */ [VALIDATION_RESULT.MEDIAN_LINE_LENGTH]: i18n.translate( 'xpack.ml.models.jobService.categorization.messages.validMedianLineLength', { defaultMessage: - 'The median line length of the examples loaded was less than {medianCharCount} characters.', + 'The median line length of the loaded examples was less than {medianCharCount} characters.', values: { medianCharCount: MEDIAN_LINE_LENGTH_LIMIT, }, } - ), + ) as string, + /** + * Less than {percentage}% of the loaded examples were null. + */ [VALIDATION_RESULT.NULL_VALUES]: i18n.translate( 'xpack.ml.models.jobService.categorization.messages.validNullValues', { - defaultMessage: 'Less than {percentage}% of the examples loaded were null.', + defaultMessage: 'Less than {percentage}% of the loaded examples were null.', values: { percentage: Math.floor(100 - NULL_COUNT_PERCENT_LIMIT * 100), }, } - ), + ) as string, + /** + * Less than 10000 tokens were found in total in the loaded examples. + */ [VALIDATION_RESULT.TOO_MANY_TOKENS]: i18n.translate( 'xpack.ml.models.jobService.categorization.messages.validTooManyTokens', { - defaultMessage: 'Less than 10000 tokens were found in total in the examples loaded.', + defaultMessage: 'Less than 10000 tokens were found in total in the loaded examples.', } - ), + ) as string, + /** + * The user has sufficient privileges to perform the checks. + */ [VALIDATION_RESULT.INSUFFICIENT_PRIVILEGES]: i18n.translate( 'xpack.ml.models.jobService.categorization.messages.validUserPrivileges', { defaultMessage: 'The user has sufficient privileges to perform the checks.', } - ), + ) as string, }; diff --git a/x-pack/packages/ml/category_validator/common/types/categories.ts b/x-pack/packages/ml/category_validator/common/types/categories.ts new file mode 100644 index 00000000000000..b54f34a6b20860 --- /dev/null +++ b/x-pack/packages/ml/category_validator/common/types/categories.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { + CATEGORY_EXAMPLES_VALIDATION_STATUS, + VALIDATION_RESULT, +} from '../constants/categorization'; + +/** + * Token + */ +export interface Token { + /** + * The token string. + */ + token: string; + /** + * The starting offset of the token. + */ + start_offset: number; + /** + * The ending offset of the token. + */ + end_offset: number; + /** + * The type of the token. + */ + type: string; + /** + * The position of the token. + */ + position: number; +} + +/** + * Categorization analyzer with additional properties. + */ +export type CategorizationAnalyzer = estypes.MlCategorizationAnalyzerDefinition & { + /** + * The analyzer used for categorization. + */ + analyzer?: string; +}; + +/** + * Field example for a category. + */ +export interface CategoryFieldExample { + /** + * The text of the field example. + */ + text: string; + /** + * The tokens extracted from the field example. + */ + tokens: Token[]; +} + +/** + * Result of a field example check. + */ +export interface FieldExampleCheck { + /** + * The ID of the validation result. + */ + id: VALIDATION_RESULT; + /** + * The validation status of the field example. + */ + valid: CATEGORY_EXAMPLES_VALIDATION_STATUS; + /** + * The message associated with the validation result. + */ + message: string; +} diff --git a/x-pack/packages/ml/category_validator/index.ts b/x-pack/packages/ml/category_validator/index.ts new file mode 100644 index 00000000000000..e235ea204ed040 --- /dev/null +++ b/x-pack/packages/ml/category_validator/index.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { categorizationExamplesProvider } from './src/examples'; +export type { + CategorizationAnalyzer, + CategoryFieldExample, + FieldExampleCheck, + Token, +} from './common/types/categories'; +export { + CATEGORY_EXAMPLES_ERROR_LIMIT, + CATEGORY_EXAMPLES_SAMPLE_SIZE, + CATEGORY_EXAMPLES_VALIDATION_STATUS, + CATEGORY_EXAMPLES_WARNING_LIMIT, + MEDIAN_LINE_LENGTH_LIMIT, + NULL_COUNT_PERCENT_LIMIT, + VALID_TOKEN_COUNT, + VALIDATION_CHECK_DESCRIPTION, + VALIDATION_RESULT, +} from './common/constants/categorization'; diff --git a/x-pack/packages/ml/category_validator/jest.config.js b/x-pack/packages/ml/category_validator/jest.config.js new file mode 100644 index 00000000000000..16140d5a9b1f7c --- /dev/null +++ b/x-pack/packages/ml/category_validator/jest.config.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test/jest_node', + rootDir: '../../../..', + roots: ['/x-pack/packages/ml/category_validator'], +}; diff --git a/x-pack/packages/ml/category_validator/kibana.jsonc b/x-pack/packages/ml/category_validator/kibana.jsonc new file mode 100644 index 00000000000000..de1fea187f3c80 --- /dev/null +++ b/x-pack/packages/ml/category_validator/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/ml-category-validator", + "owner": "@elastic/ml-ui" +} diff --git a/x-pack/packages/ml/category_validator/package.json b/x-pack/packages/ml/category_validator/package.json new file mode 100644 index 00000000000000..2466a5fdc6b7d8 --- /dev/null +++ b/x-pack/packages/ml/category_validator/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/ml-category-validator", + "private": true, + "version": "1.0.0", + "license": "Elastic License 2.0" +} diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/examples.ts b/x-pack/packages/ml/category_validator/src/examples.ts similarity index 83% rename from x-pack/plugins/ml/server/models/job_service/new_job/categorization/examples.ts rename to x-pack/packages/ml/category_validator/src/examples.ts index f529b94da5e84e..7971e825d3fa56 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/examples.ts +++ b/x-pack/packages/ml/category_validator/src/examples.ts @@ -7,29 +7,53 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { IScopedClusterClient } from '@kbn/core/server'; import { chunk } from 'lodash'; +import type { IScopedClusterClient } from '@kbn/core/server'; import type { RuntimeMappings } from '@kbn/ml-runtime-field-utils'; -import { CATEGORY_EXAMPLES_SAMPLE_SIZE } from '../../../../../common/constants/categorization_job'; -import { +import { CATEGORY_EXAMPLES_SAMPLE_SIZE } from '../common/constants/categorization'; +import type { Token, CategorizationAnalyzer, CategoryFieldExample, -} from '../../../../../common/types/categories'; -import { IndicesOptions } from '../../../../../common/types/anomaly_detection_jobs'; +} from '../common/types/categories'; import { ValidationResults } from './validation_results'; +/** + * The size of each chunk for processing examples. + * + */ const CHUNK_SIZE = 100; -export function categorizationExamplesProvider({ - asCurrentUser, - asInternalUser, -}: IScopedClusterClient) { +/** + * Provides methods for checking whether categories can be + * produced from a field. + * + * @export + * @param client - IScopedClusterClient + */ +export function categorizationExamplesProvider(client: IScopedClusterClient) { + const { asCurrentUser, asInternalUser } = client; const validationResults = new ValidationResults(); + /** + * Retrieves the tokens for the provided examples and analyzer. + * + * @async + * @param {string} indexPatternTitle + * @param {estypes.QueryDslQueryContainer} query + * @param {number} size + * @param {string} categorizationFieldName + * @param {(string | undefined)} timeField + * @param {number} start + * @param {number} end + * @param {CategorizationAnalyzer} analyzer + * @param {(RuntimeMappings | undefined)} runtimeMappings + * @param {(estypes.IndicesOptions | undefined)} indicesOptions + * @returns {Promise<{ examples: CategoryFieldExample[]; error?: Error }>} + */ async function categorizationExamples( indexPatternTitle: string, - query: any, + query: estypes.QueryDslQueryContainer, size: number, categorizationFieldName: string, timeField: string | undefined, @@ -37,8 +61,8 @@ export function categorizationExamplesProvider({ end: number, analyzer: CategorizationAnalyzer, runtimeMappings: RuntimeMappings | undefined, - indicesOptions: IndicesOptions | undefined - ): Promise<{ examples: CategoryFieldExample[]; error?: any }> { + indicesOptions: estypes.IndicesOptions | undefined + ): Promise<{ examples: CategoryFieldExample[]; error?: Error }> { if (timeField !== undefined) { const range = { range: { @@ -178,7 +202,7 @@ export function categorizationExamplesProvider({ async function validateCategoryExamples( indexPatternTitle: string, - query: any, + query: estypes.QueryDslQueryContainer, size: number, categorizationFieldName: string, timeField: string | undefined, @@ -186,7 +210,7 @@ export function categorizationExamplesProvider({ end: number, analyzer: CategorizationAnalyzer, runtimeMappings: RuntimeMappings | undefined, - indicesOptions: IndicesOptions | undefined + indicesOptions: estypes.IndicesOptions | undefined ) { const resp = await categorizationExamples( indexPatternTitle, diff --git a/x-pack/packages/ml/category_validator/src/util.ts b/x-pack/packages/ml/category_validator/src/util.ts new file mode 100644 index 00000000000000..8bcbc38e065abe --- /dev/null +++ b/x-pack/packages/ml/category_validator/src/util.ts @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export function getMedianStringLength(strings: string[]) { + const sortedStringLengths = strings.map((s) => s.length).sort((a, b) => a - b); + return sortedStringLengths[Math.floor(sortedStringLengths.length / 2)] || 0; +} diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/validation_results.ts b/x-pack/packages/ml/category_validator/src/validation_results.ts similarity index 96% rename from x-pack/plugins/ml/server/models/job_service/new_job/categorization/validation_results.ts rename to x-pack/packages/ml/category_validator/src/validation_results.ts index a5617f1fc7b659..3d6ab9623a869a 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/validation_results.ts +++ b/x-pack/packages/ml/category_validator/src/validation_results.ts @@ -13,13 +13,10 @@ import { CATEGORY_EXAMPLES_VALIDATION_STATUS, CATEGORY_EXAMPLES_ERROR_LIMIT, CATEGORY_EXAMPLES_WARNING_LIMIT, -} from '../../../../../common/constants/categorization_job'; -import { - FieldExampleCheck, - CategoryFieldExample, VALIDATION_RESULT, -} from '../../../../../common/types/categories'; -import { getMedianStringLength } from '../../../../../common/util/string_utils'; +} from '../common/constants/categorization'; +import type { FieldExampleCheck, CategoryFieldExample } from '../common/types/categories'; +import { getMedianStringLength } from './util'; export class ValidationResults { private _results: FieldExampleCheck[] = []; diff --git a/x-pack/packages/ml/category_validator/tsconfig.json b/x-pack/packages/ml/category_validator/tsconfig.json new file mode 100644 index 00000000000000..51c0faec5b5c05 --- /dev/null +++ b/x-pack/packages/ml/category_validator/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts", + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/i18n", + "@kbn/core", + "@kbn/ml-runtime-field-utils", + ] +} diff --git a/x-pack/plugins/ml/common/constants/new_job.ts b/x-pack/plugins/ml/common/constants/new_job.ts index 35f9dea4135e02..12aa5393ad6e15 100644 --- a/x-pack/plugins/ml/common/constants/new_job.ts +++ b/x-pack/plugins/ml/common/constants/new_job.ts @@ -33,3 +33,5 @@ export const DEFAULT_RARE_BUCKET_SPAN = '1h'; export const DEFAULT_QUERY_DELAY = '60s'; export const SHARED_RESULTS_INDEX_NAME = 'shared'; + +export const NUMBER_OF_CATEGORY_EXAMPLES = 5; diff --git a/x-pack/plugins/ml/common/types/categories.ts b/x-pack/plugins/ml/common/types/categories.ts index c40b3035a5f817..3b396524207e84 100644 --- a/x-pack/plugins/ml/common/types/categories.ts +++ b/x-pack/plugins/ml/common/types/categories.ts @@ -5,8 +5,6 @@ * 2.0. */ -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '../constants/categorization_job'; - export type CategoryId = number; export interface Category { @@ -20,39 +18,3 @@ export interface Category { partition_field_name?: string; // TODO: make non-optional once fields have been added to the results partition_field_value?: string; // TODO: make non-optional once fields have been added to the results } - -export interface Token { - token: string; - start_offset: number; - end_offset: number; - type: string; - position: number; -} - -export interface CategorizationAnalyzer { - char_filter?: any[]; - tokenizer?: string; - filter?: any[]; - analyzer?: string; -} - -export interface CategoryFieldExample { - text: string; - tokens: Token[]; -} - -export enum VALIDATION_RESULT { - NO_EXAMPLES, - FAILED_TO_TOKENIZE, - TOO_MANY_TOKENS, - TOKEN_COUNT, - MEDIAN_LINE_LENGTH, - NULL_VALUES, - INSUFFICIENT_PRIVILEGES, -} - -export interface FieldExampleCheck { - id: VALIDATION_RESULT; - valid: CATEGORY_EXAMPLES_VALIDATION_STATUS; - message: string; -} diff --git a/x-pack/plugins/ml/common/types/ml_server_info.ts b/x-pack/plugins/ml/common/types/ml_server_info.ts index 2886257e35e678..e5141d6f2e78fb 100644 --- a/x-pack/plugins/ml/common/types/ml_server_info.ts +++ b/x-pack/plugins/ml/common/types/ml_server_info.ts @@ -5,14 +5,14 @@ * 2.0. */ -import { CategorizationAnalyzer } from './categories'; +import * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export interface MlServerDefaults { anomaly_detectors: { categorization_examples_limit?: number; model_memory_limit?: string; model_snapshot_retention_days?: number; - categorization_analyzer?: CategorizationAnalyzer; + categorization_analyzer?: estypes.MlCategorizationAnalyzerDefinition; }; datafeeds: { scroll_size?: number }; } diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts index 72ff937a8239f7..0ed702f80d472f 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/categorization_job_creator.ts @@ -14,8 +14,19 @@ import { ML_JOB_AGGREGATION, } from '@kbn/ml-anomaly-utils'; import type { SavedSearch } from '@kbn/saved-search-plugin/public'; +import { + type CategorizationAnalyzer, + type CategoryFieldExample, + type FieldExampleCheck, + VALIDATION_RESULT, + CATEGORY_EXAMPLES_VALIDATION_STATUS, +} from '@kbn/ml-category-validator'; import { JobCreator } from './job_creator'; -import { Job, Datafeed, Detector } from '../../../../../../common/types/anomaly_detection_jobs'; +import type { + Job, + Datafeed, + Detector, +} from '../../../../../../common/types/anomaly_detection_jobs'; import { createBasicDetector } from './util/default_configs'; import { JOB_TYPE, @@ -23,13 +34,7 @@ import { DEFAULT_BUCKET_SPAN, DEFAULT_RARE_BUCKET_SPAN, } from '../../../../../../common/constants/new_job'; -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '../../../../../../common/constants/categorization_job'; -import { - CategorizationAnalyzer, - CategoryFieldExample, - FieldExampleCheck, - VALIDATION_RESULT, -} from '../../../../../../common/types/categories'; + import { getRichDetectors } from './util/general'; import { CategorizationExamplesLoader } from '../results_loader'; import { getNewJobDefaults } from '../../../../services/ml_server_info'; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts index 812bdb0b935b57..da3ad4b1216a53 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/job_validator/job_validator.ts @@ -8,6 +8,7 @@ import { ReactElement } from 'react'; import { combineLatest, Observable, ReplaySubject, Subject } from 'rxjs'; import { map, startWith, tap } from 'rxjs/operators'; +import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-category-validator'; import { basicJobValidation, basicDatafeedValidation, @@ -24,7 +25,7 @@ import { JobExistsResult, GroupsExistResult, } from './validators'; -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '../../../../../../common/constants/categorization_job'; + import { JOB_TYPE } from '../../../../../../common/constants/new_job'; // delay start of validation to allow the user to make changes diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts index 7ea7d60706f847..bc7a6edb17a380 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts +++ b/x-pack/plugins/ml/public/application/jobs/new_job/common/results_loader/categorization_examples_loader.ts @@ -6,13 +6,11 @@ */ import type { DataView } from '@kbn/data-views-plugin/public'; -import { IndexPatternTitle } from '../../../../../../common/types/kibana'; +import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-category-validator'; +import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../common/constants/new_job'; +import type { IndexPatternTitle } from '../../../../../../common/types/kibana'; import { CategorizationJobCreator } from '../job_creator'; import { ml } from '../../../../services/ml_api_service'; -import { - NUMBER_OF_CATEGORY_EXAMPLES, - CATEGORY_EXAMPLES_VALIDATION_STATUS, -} from '../../../../../../common/constants/categorization_job'; export class CategorizationExamplesLoader { private _jobCreator: CategorizationJobCreator; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx index 4d22d8c4a5f507..1d30605b4098c7 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/examples_valid_callout.tsx @@ -18,15 +18,14 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; import { - CategorizationAnalyzer, - FieldExampleCheck, -} from '../../../../../../../../../common/types/categories'; -import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; -import { + type CategorizationAnalyzer, + type FieldExampleCheck, + VALIDATION_RESULT, CATEGORY_EXAMPLES_VALIDATION_STATUS, VALIDATION_CHECK_DESCRIPTION, -} from '../../../../../../../../../common/constants/categorization_job'; -import { VALIDATION_RESULT } from '../../../../../../../../../common/types/categories'; +} from '@kbn/ml-category-validator'; + +import { EditCategorizationAnalyzerFlyout } from '../../../common/edit_categorization_analyzer_flyout'; interface Props { validationChecks: FieldExampleCheck[]; @@ -96,9 +95,12 @@ export const ExamplesValidCallout: FC = ({ const AnalyzerUsed: FC<{ categorizationAnalyzer: CategorizationAnalyzer }> = ({ categorizationAnalyzer, }) => { - let analyzer = ''; + let analyzer: string | null = null; - if (categorizationAnalyzer?.tokenizer !== undefined) { + if ( + categorizationAnalyzer?.tokenizer !== undefined && + typeof categorizationAnalyzer.tokenizer === 'string' + ) { analyzer = categorizationAnalyzer.tokenizer; } else if (categorizationAnalyzer?.analyzer !== undefined) { analyzer = categorizationAnalyzer.analyzer; @@ -106,13 +108,15 @@ const AnalyzerUsed: FC<{ categorizationAnalyzer: CategorizationAnalyzer }> = ({ return ( <> -
- -
+ {analyzer !== null ? ( +
+ +
+ ) : null}
diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx index 01eee1c9cc37ba..5b00a812301513 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/field_examples.tsx @@ -8,7 +8,7 @@ import React, { FC } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiBasicTable, EuiCodeBlock } from '@elastic/eui'; -import { CategoryFieldExample } from '../../../../../../../../../common/types/categories'; +import { CategoryFieldExample } from '@kbn/ml-category-validator'; interface Props { fieldExamples: CategoryFieldExample[] | null; diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx index 1b89262c9e59f0..7b0f48087940b2 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/metric_selection.tsx @@ -7,6 +7,11 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { EuiHorizontalRule } from '@elastic/eui'; +import { + CATEGORY_EXAMPLES_VALIDATION_STATUS, + type CategoryFieldExample, + type FieldExampleCheck, +} from '@kbn/ml-category-validator'; import { getToastNotificationService } from '../../../../../../../services/toast_notification_service'; import { JobCreatorContext } from '../../../job_creator_context'; @@ -18,11 +23,6 @@ import { CategorizationPerPartitionField } from '../categorization_partition_fie import { FieldExamples } from './field_examples'; import { ExamplesValidCallout } from './examples_valid_callout'; import { InvalidCssVersionCallout } from './invalid_ccs_version_valid_callout'; -import { - CategoryFieldExample, - FieldExampleCheck, -} from '../../../../../../../../../common/types/categories'; -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '../../../../../../../../../common/constants/categorization_job'; import { LoadingWrapper } from '../../../charts/loading_wrapper'; interface Props { diff --git a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx index b12b4261a0628b..6ce1569794c3d6 100644 --- a/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx +++ b/x-pack/plugins/ml/public/application/jobs/new_job/pages/components/pick_fields_step/components/categorization_view/top_categories.tsx @@ -9,12 +9,12 @@ import React, { FC, useContext, useEffect, useState } from 'react'; import { EuiBasicTable, EuiText } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n-react'; import { extractErrorProperties } from '@kbn/ml-error-utils'; +import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../../../../common/constants/new_job'; import { JobCreatorContext } from '../../../job_creator_context'; import { CategorizationJobCreator } from '../../../../../common/job_creator'; import { Results } from '../../../../../common/results_loader'; import { ml } from '../../../../../../../services/ml_api_service'; import { useToastNotificationService } from '../../../../../../../services/toast_notification_service'; -import { NUMBER_OF_CATEGORY_EXAMPLES } from '../../../../../../../../../common/constants/categorization_job'; export const TopCategories: FC = () => { const { displayErrorToast } = useToastNotificationService(); diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts index 98a1246b165868..f84d030654ab2b 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts @@ -9,6 +9,12 @@ import { Observable } from 'rxjs'; import { useMemo } from 'react'; import type { AggFieldNamePair } from '@kbn/ml-anomaly-utils'; import type { RuntimeMappings } from '@kbn/ml-runtime-field-utils'; +import { + type CategorizationAnalyzer, + type CategoryFieldExample, + type FieldExampleCheck, + CATEGORY_EXAMPLES_VALIDATION_STATUS, +} from '@kbn/ml-category-validator'; import { HttpService } from '../http_service'; import { useMlKibana } from '../../contexts/kibana'; @@ -25,12 +31,7 @@ import type { JobMessage } from '../../../../common/types/audit_message'; import type { JobAction } from '../../../../common/constants/job_actions'; import type { Group } from '../../../../common/types/groups'; import type { ExistingJobsAndGroups } from '../job_service'; -import type { - CategorizationAnalyzer, - CategoryFieldExample, - FieldExampleCheck, -} from '../../../../common/types/categories'; -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '../../../../common/constants/categorization_job'; + import type { Category } from '../../../../common/types/categories'; import type { JobsExistResponse, diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/index.ts b/x-pack/plugins/ml/server/models/job_service/new_job/categorization/index.ts index c44a66a9ada20a..38c35b0e76a2ce 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/index.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job/categorization/index.ts @@ -5,5 +5,4 @@ * 2.0. */ -export { categorizationExamplesProvider } from './examples'; export { topCategoriesProvider } from './top_categories'; diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/top_categories.ts b/x-pack/plugins/ml/server/models/job_service/new_job/categorization/top_categories.ts index 8d019137f2ff36..339ba87fc423b6 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/categorization/top_categories.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job/categorization/top_categories.ts @@ -7,7 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { CategoryId, Category } from '../../../../../common/types/categories'; +import type { CategoryId, Category } from '../../../../../common/types/categories'; import type { MlClient } from '../../../../lib/ml_client'; export function topCategoriesProvider(mlClient: MlClient) { diff --git a/x-pack/plugins/ml/server/models/job_service/new_job/index.ts b/x-pack/plugins/ml/server/models/job_service/new_job/index.ts index 9f51d48cbd2f62..2550510aa66105 100644 --- a/x-pack/plugins/ml/server/models/job_service/new_job/index.ts +++ b/x-pack/plugins/ml/server/models/job_service/new_job/index.ts @@ -6,4 +6,4 @@ */ export { newJobChartsProvider } from './charts'; -export { categorizationExamplesProvider, topCategoriesProvider } from './categorization'; +export { topCategoriesProvider } from './categorization'; diff --git a/x-pack/plugins/ml/server/routes/job_service.ts b/x-pack/plugins/ml/server/routes/job_service.ts index cb8bbebd95d31b..c7cdd2276116df 100644 --- a/x-pack/plugins/ml/server/routes/job_service.ts +++ b/x-pack/plugins/ml/server/routes/job_service.ts @@ -7,6 +7,7 @@ import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { schema } from '@kbn/config-schema'; +import { categorizationExamplesProvider } from '@kbn/ml-category-validator'; import { ML_INTERNAL_BASE_PATH } from '../../common/constants/app'; import { wrapError } from '../client/error_wrapper'; import type { RouteInitialization } from '../types'; @@ -32,7 +33,6 @@ import { import { jobIdSchema } from './schemas/anomaly_detectors_schema'; import { jobServiceProvider } from '../models/job_service'; -import { categorizationExamplesProvider } from '../models/job_service/new_job'; import { getAuthorizationHeader } from '../lib/request_authorization'; import type { Datafeed, Job } from '../../common/types/anomaly_detection_jobs'; diff --git a/x-pack/plugins/ml/tsconfig.json b/x-pack/plugins/ml/tsconfig.json index 98100a0b9bf0e9..441d3814496bae 100644 --- a/x-pack/plugins/ml/tsconfig.json +++ b/x-pack/plugins/ml/tsconfig.json @@ -95,6 +95,7 @@ "@kbn/ml-kibana-theme", "@kbn/ml-runtime-field-utils", "@kbn/ml-date-utils", + "@kbn/ml-category-validator", "@kbn/deeplinks-ml", "@kbn/core-notifications-browser-mocks", "@kbn/unified-field-list", diff --git a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/categorization_job.ts b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/categorization_job.ts index 2bf433a7d77c38..b1939d75d05c31 100644 --- a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/categorization_job.ts +++ b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/categorization_job.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-plugin/common/constants/categorization_job'; +import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-category-validator'; import type { FtrProviderContext } from '../../../ftr_provider_context'; import type { FieldStatsType } from '../common/types'; diff --git a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/convert_jobs_to_advanced_job.ts b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/convert_jobs_to_advanced_job.ts index 230f2d79f24486..49383315df1f9a 100644 --- a/x-pack/test/functional/apps/ml/anomaly_detection_jobs/convert_jobs_to_advanced_job.ts +++ b/x-pack/test/functional/apps/ml/anomaly_detection_jobs/convert_jobs_to_advanced_job.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-plugin/common/constants/categorization_job'; +import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-category-validator'; import type { PickFieldsConfig, DatafeedConfig, Detector } from './types'; import type { FtrProviderContext } from '../../../ftr_provider_context'; diff --git a/x-pack/test/functional/services/ml/job_wizard_categorization.ts b/x-pack/test/functional/services/ml/job_wizard_categorization.ts index a646106c422535..49d37a0db42147 100644 --- a/x-pack/test/functional/services/ml/job_wizard_categorization.ts +++ b/x-pack/test/functional/services/ml/job_wizard_categorization.ts @@ -7,7 +7,7 @@ import expect from '@kbn/expect'; -import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-plugin/common/constants/categorization_job'; +import { CATEGORY_EXAMPLES_VALIDATION_STATUS } from '@kbn/ml-category-validator'; import type { FtrProviderContext } from '../../ftr_provider_context'; import type { MlCommonFieldStatsFlyout } from './field_stats_flyout'; diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 5cda7fbbdf8fdd..8d7ef58f2775ea 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -135,6 +135,7 @@ "@kbn/profiling-plugin", "@kbn/observability-onboarding-plugin", "@kbn/bfetch-plugin", - "@kbn/uptime-plugin" + "@kbn/uptime-plugin", + "@kbn/ml-category-validator" ] } diff --git a/yarn.lock b/yarn.lock index 5a26fe36af99c5..91a7de87818a18 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4658,6 +4658,10 @@ version "0.0.0" uid "" +"@kbn/ml-category-validator@link:x-pack/packages/ml/category_validator": + version "0.0.0" + uid "" + "@kbn/ml-data-frame-analytics-utils@link:x-pack/packages/ml/data_frame_analytics_utils": version "0.0.0" uid ""