From e3136aa83028236a9e118b345766681f5e988c36 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 27 Mar 2024 16:33:13 +0100 Subject: [PATCH 01/31] plugin boilerplate --- package.json | 1 + tsconfig.base.json | 2 + x-pack/packages/ml/aiops_common/constants.ts | 5 ++ x-pack/plugins/aiops-api/README.md | 3 + x-pack/plugins/aiops-api/jest.config.js | 15 ++++ x-pack/plugins/aiops-api/kibana.jsonc | 16 ++++ x-pack/plugins/aiops-api/server/index.ts | 15 ++++ .../plugins/aiops-api/server/lib/license.ts | 12 +++ x-pack/plugins/aiops-api/server/plugin.ts | 81 +++++++++++++++++++ .../routes/log_rate_analysis/define_route.ts | 48 +++++++++++ .../route_handler_factory.ts | 63 +++++++++++++++ x-pack/plugins/aiops-api/server/types.ts | 38 +++++++++ x-pack/plugins/aiops-api/tsconfig.json | 17 ++++ yarn.lock | 4 + 14 files changed, 320 insertions(+) create mode 100755 x-pack/plugins/aiops-api/README.md create mode 100644 x-pack/plugins/aiops-api/jest.config.js create mode 100644 x-pack/plugins/aiops-api/kibana.jsonc create mode 100755 x-pack/plugins/aiops-api/server/index.ts create mode 100644 x-pack/plugins/aiops-api/server/lib/license.ts create mode 100755 x-pack/plugins/aiops-api/server/plugin.ts create mode 100644 x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts create mode 100644 x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts create mode 100755 x-pack/plugins/aiops-api/server/types.ts create mode 100644 x-pack/plugins/aiops-api/tsconfig.json diff --git a/package.json b/package.json index e9a8d26a23c8e1..1acaf2d551e9b4 100644 --- a/package.json +++ b/package.json @@ -148,6 +148,7 @@ "@kbn/actions-types": "link:packages/kbn-actions-types", "@kbn/advanced-settings-plugin": "link:src/plugins/advanced_settings", "@kbn/ai-assistant-management-plugin": "link:src/plugins/ai_assistant_management/selection", + "@kbn/aiops-api-plugin": "link:x-pack/plugins/aiops-api", "@kbn/aiops-change-point-detection": "link:x-pack/packages/ml/aiops_change_point_detection", "@kbn/aiops-common": "link:x-pack/packages/ml/aiops_common", "@kbn/aiops-components": "link:x-pack/packages/ml/aiops_components", diff --git a/tsconfig.base.json b/tsconfig.base.json index 85f84a2609046b..1e0bf30c5fa868 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -18,6 +18,8 @@ "@kbn/advanced-settings-plugin/*": ["src/plugins/advanced_settings/*"], "@kbn/ai-assistant-management-plugin": ["src/plugins/ai_assistant_management/selection"], "@kbn/ai-assistant-management-plugin/*": ["src/plugins/ai_assistant_management/selection/*"], + "@kbn/aiops-api-plugin": ["x-pack/plugins/aiops-api"], + "@kbn/aiops-api-plugin/*": ["x-pack/plugins/aiops-api/*"], "@kbn/aiops-change-point-detection": ["x-pack/packages/ml/aiops_change_point_detection"], "@kbn/aiops-change-point-detection/*": ["x-pack/packages/ml/aiops_change_point_detection/*"], "@kbn/aiops-common": ["x-pack/packages/ml/aiops_common"], diff --git a/x-pack/packages/ml/aiops_common/constants.ts b/x-pack/packages/ml/aiops_common/constants.ts index dd5b0b1fd034c5..5eee02aa34bc0a 100644 --- a/x-pack/packages/ml/aiops_common/constants.ts +++ b/x-pack/packages/ml/aiops_common/constants.ts @@ -10,6 +10,11 @@ */ export const AIOPS_PLUGIN_ID = 'aiops'; +/** + * AIOPS_API_PLUGIN_ID is used as a unique identifier for the aiops API plugin for public APIs. + */ +export const AIOPS_API_PLUGIN_ID = 'aiops'; + export const AIOPS_API_ENDPOINT = { LOG_RATE_ANALYSIS: '/internal/aiops/log_rate_analysis', CATEGORIZATION_FIELD_VALIDATION: '/internal/aiops/categorization_field_validation', diff --git a/x-pack/plugins/aiops-api/README.md b/x-pack/plugins/aiops-api/README.md new file mode 100755 index 00000000000000..d151ddf72bdb0e --- /dev/null +++ b/x-pack/plugins/aiops-api/README.md @@ -0,0 +1,3 @@ +# aiops-api + +The plugin provides public APIs for AIOps features. diff --git a/x-pack/plugins/aiops-api/jest.config.js b/x-pack/plugins/aiops-api/jest.config.js new file mode 100644 index 00000000000000..3c066ce0cd1d73 --- /dev/null +++ b/x-pack/plugins/aiops-api/jest.config.js @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +module.exports = { + preset: '@kbn/test', + rootDir: '../../..', + roots: ['/x-pack/plugins/aiops-api'], + coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/aiops-api', + coverageReporters: ['text', 'html'], + collectCoverageFrom: ['/x-pack/plugins/aiops-api/server/**/*.{js,ts,tsx}'], +}; diff --git a/x-pack/plugins/aiops-api/kibana.jsonc b/x-pack/plugins/aiops-api/kibana.jsonc new file mode 100644 index 00000000000000..e5a3ba841ba90a --- /dev/null +++ b/x-pack/plugins/aiops-api/kibana.jsonc @@ -0,0 +1,16 @@ +{ + "type": "plugin", + "id": "@kbn/aiops-api-plugin", + "owner": "@elastic/ml-ui", + "description": "AIOps public API plugin maintained by ML team.", + "plugin": { + "id": "aiopsApi", + "server": true, + "browser": false, + "requiredPlugins": [ + "licensing", + ], + "optionalPlugins": [], + "requiredBundles": [] + } +} diff --git a/x-pack/plugins/aiops-api/server/index.ts b/x-pack/plugins/aiops-api/server/index.ts new file mode 100755 index 00000000000000..f080e8fe9a3ead --- /dev/null +++ b/x-pack/plugins/aiops-api/server/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginInitializerContext } from '@kbn/core/server'; + +export async function plugin(initializerContext: PluginInitializerContext) { + const { AiopsApiPlugin } = await import('./plugin'); + return new AiopsApiPlugin(initializerContext); +} + +export type { AiopsApiPluginSetup, AiopsApiPluginStart } from './types'; diff --git a/x-pack/plugins/aiops-api/server/lib/license.ts b/x-pack/plugins/aiops-api/server/lib/license.ts new file mode 100644 index 00000000000000..6e2173ab21ef8e --- /dev/null +++ b/x-pack/plugins/aiops-api/server/lib/license.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ILicense, LicenseType } from '@kbn/licensing-plugin/common/types'; + +export function isActiveLicense(licenseType: LicenseType, license?: ILicense): boolean { + return (license && license.isActive && license.hasAtLeast(licenseType)) || false; +} diff --git a/x-pack/plugins/aiops-api/server/plugin.ts b/x-pack/plugins/aiops-api/server/plugin.ts new file mode 100755 index 00000000000000..1f1842cdcf43e0 --- /dev/null +++ b/x-pack/plugins/aiops-api/server/plugin.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { Subscription } from 'rxjs'; + +import type { + PluginInitializerContext, + CoreSetup, + CoreStart, + Plugin, + Logger, +} from '@kbn/core/server'; +import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; +import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; +import { AIOPS_API_PLUGIN_ID } from '@kbn/aiops-common/constants'; +import { isActiveLicense } from './lib/license'; +import type { + AiopsApiLicense, + AiopsApiPluginSetup, + AiopsApiPluginStart, + AiopsApiPluginSetupDeps, + AiopsApiPluginStartDeps, +} from './types'; +import { defineRoute as defineLogRateAnalysisRoute } from './routes/log_rate_analysis/define_route'; + +export class AiopsApiPlugin + implements + Plugin< + AiopsApiPluginSetup, + AiopsApiPluginStart, + AiopsApiPluginSetupDeps, + AiopsApiPluginStartDeps + > +{ + private readonly logger: Logger; + private licenseSubscription: Subscription | null = null; + private usageCounter?: UsageCounter; + + constructor(initializerContext: PluginInitializerContext) { + this.logger = initializerContext.logger.get(); + } + + public setup( + core: CoreSetup, + plugins: AiopsApiPluginSetupDeps + ) { + this.logger.debug('aiops API: Setup'); + this.usageCounter = plugins.usageCollection?.createUsageCounter(AIOPS_API_PLUGIN_ID); + + // Subscribe to license changes and store the current license in `currentLicense`. + // This way we can pass on license changes to the route factory having always + // the current license because it's stored in a mutable attribute. + const aiopsLicense: AiopsApiLicense = { isActivePlatinumLicense: false }; + this.licenseSubscription = plugins.licensing.license$.subscribe((license) => { + aiopsLicense.isActivePlatinumLicense = isActiveLicense('platinum', license); + }); + + const router = core.http.createRouter(); + + // Register server side APIs + core.getStartServices().then(([coreStart, depsStart]) => { + defineLogRateAnalysisRoute(router, aiopsLicense, this.logger, coreStart, this.usageCounter); + }); + + return {}; + } + + public start(core: CoreStart) { + this.logger.debug('aiops API: Started'); + return {}; + } + + public stop() { + this.logger.debug('aiops API: Stop'); + this.licenseSubscription?.unsubscribe(); + } +} diff --git a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts new file mode 100644 index 00000000000000..16a5f247c2e978 --- /dev/null +++ b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreStart, IRouter } from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; +import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; +import { aiopsLogRateAnalysisSchemaV1 } from '@kbn/aiops-log-rate-analysis/api/schema_v1'; +// import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants'; + +import type { AiopsApiLicense } from '../../types'; + +import { routeHandlerFactory } from './route_handler_factory'; + +/** + * `defineRoute` is called in the root `plugin.ts` to set up the API route + * for log pattern analysis. Its purpose is to take care of the route setup + * and versioning only. `routeHandlerFactory` is used to take care of + * the actual route logic. + */ +export const defineRoute = ( + router: IRouter, + license: AiopsApiLicense, + logger: Logger, + coreStart: CoreStart, + usageCounter?: UsageCounter +) => { + router.versioned + .post({ + path: '/api/aiops/log_rate_analysis', + access: 'public', + }) + .addVersion( + { + version: '2023-10-31', + validate: { + request: { + body: aiopsLogRateAnalysisSchemaV1, + }, + }, + }, + routeHandlerFactory('1', license, logger, coreStart, usageCounter) + ); +}; diff --git a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts new file mode 100644 index 00000000000000..2410e5690d3014 --- /dev/null +++ b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { + CoreStart, + KibanaRequest, + RequestHandlerContext, + RequestHandler, + KibanaResponseFactory, +} from '@kbn/core/server'; +import type { Logger } from '@kbn/logging'; +// import { createExecutionContext } from '@kbn/ml-route-utils'; +import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; +// import { AIOPS_TELEMETRY_ID, AIOPS_PLUGIN_ID } from '@kbn/aiops-common/constants'; +import type { + AiopsLogRateAnalysisSchema, + AiopsLogRateAnalysisApiVersion as ApiVersion, +} from '@kbn/aiops-log-rate-analysis/api/schema'; +// import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants'; +// import { isRequestAbortedError } from '@kbn/aiops-common/is_request_aborted_error'; + +// import { trackAIOpsRouteUsage } from '../../lib/track_route_usage'; +import type { AiopsApiLicense } from '../../types'; + +/** + * The log rate analysis route handler sets up `responseStreamFactory` + * to create the response stream and then uses its handlers to + * walk through the steps of the analysis. + */ +export function routeHandlerFactory( + version: T, + license: AiopsApiLicense, + logger: Logger, + coreStart: CoreStart, + usageCounter?: UsageCounter +): RequestHandler> { + return async ( + context: RequestHandlerContext, + request: KibanaRequest>, + response: KibanaResponseFactory + ) => { + // const { headers } = request; + + // trackAIOpsRouteUsage( + // `POST ${AIOPS_API_ENDPOINT.LOG_RATE_ANALYSIS}`, + // headers[AIOPS_TELEMETRY_ID.AIOPS_ANALYSIS_RUN_ORIGIN], + // usageCounter + // ); + + if (!license.isActivePlatinumLicense) { + return response.forbidden(); + } + + // const client = (await context.core).elasticsearch.client.asCurrentUser; + // const executionContext = createExecutionContext(coreStart, AIOPS_PLUGIN_ID, request.route.path); + + return response.ok(); + }; +} diff --git a/x-pack/plugins/aiops-api/server/types.ts b/x-pack/plugins/aiops-api/server/types.ts new file mode 100755 index 00000000000000..00081e95d5374c --- /dev/null +++ b/x-pack/plugins/aiops-api/server/types.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { PluginSetup, PluginStart } from '@kbn/data-plugin/server'; +import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; +import type { CasesServerSetup } from '@kbn/cases-plugin/server'; +import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; + +export interface AiopsApiPluginSetupDeps { + data: PluginSetup; + licensing: LicensingPluginSetup; + cases?: CasesServerSetup; + usageCollection?: UsageCollectionSetup; +} + +export interface AiopsApiPluginStartDeps { + data: PluginStart; +} + +/** + * aiops API plugin server setup contract + */ +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface AiopsApiPluginSetup {} + +/** + * aiops API plugin server start contract + */ +// eslint-disable-next-line @typescript-eslint/no-empty-interface +export interface AiopsApiPluginStart {} + +export interface AiopsApiLicense { + isActivePlatinumLicense: boolean; +} diff --git a/x-pack/plugins/aiops-api/tsconfig.json b/x-pack/plugins/aiops-api/tsconfig.json new file mode 100644 index 00000000000000..fe83c696e834e5 --- /dev/null +++ b/x-pack/plugins/aiops-api/tsconfig.json @@ -0,0 +1,17 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + }, + "include": [ + "../../../typings/**/*", + "public/**/*", + "scripts/**/*", + "server/**/*", + "types/**/*" + ], + "kbn_references": [], + "exclude": [ + "target/**/*", + ] +} diff --git a/yarn.lock b/yarn.lock index 67e950497963ba..cd630f959981a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3249,6 +3249,10 @@ version "0.0.0" uid "" +"@kbn/aiops-api-plugin@link:x-pack/plugins/aiops-api": + version "0.0.0" + uid "" + "@kbn/aiops-change-point-detection@link:x-pack/packages/ml/aiops_change_point_detection": version "0.0.0" uid "" From 98bf7187fc49d393e3f16e48d7cd49905a534c91 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 27 Mar 2024 20:34:39 +0100 Subject: [PATCH 02/31] change to non-versioned API --- .../routes/log_rate_analysis/define_route.ts | 25 ++++++------------- .../route_handler_factory.ts | 6 ++++- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts index 16a5f247c2e978..beced85b93a72b 100644 --- a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts +++ b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts @@ -9,7 +9,7 @@ import type { CoreStart, IRouter } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; import type { DataRequestHandlerContext } from '@kbn/data-plugin/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; -import { aiopsLogRateAnalysisSchemaV1 } from '@kbn/aiops-log-rate-analysis/api/schema_v1'; +// import { aiopsLogRateAnalysisSchemaV1 } from '@kbn/aiops-log-rate-analysis/api/schema_v1'; // import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants'; import type { AiopsApiLicense } from '../../types'; @@ -18,7 +18,7 @@ import { routeHandlerFactory } from './route_handler_factory'; /** * `defineRoute` is called in the root `plugin.ts` to set up the API route - * for log pattern analysis. Its purpose is to take care of the route setup + * for log rate analysis. Its purpose is to take care of the route setup * and versioning only. `routeHandlerFactory` is used to take care of * the actual route logic. */ @@ -29,20 +29,11 @@ export const defineRoute = ( coreStart: CoreStart, usageCounter?: UsageCounter ) => { - router.versioned - .post({ + router.get( + { path: '/api/aiops/log_rate_analysis', - access: 'public', - }) - .addVersion( - { - version: '2023-10-31', - validate: { - request: { - body: aiopsLogRateAnalysisSchemaV1, - }, - }, - }, - routeHandlerFactory('1', license, logger, coreStart, usageCounter) - ); + validate: false, + }, + routeHandlerFactory('1', license, logger, coreStart, usageCounter) + ); }; diff --git a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts index 2410e5690d3014..55850385deb1a1 100644 --- a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -58,6 +58,10 @@ export function routeHandlerFactory( // const client = (await context.core).elasticsearch.client.asCurrentUser; // const executionContext = createExecutionContext(coreStart, AIOPS_PLUGIN_ID, request.route.path); - return response.ok(); + return response.ok({ + body: { + time: new Date().toISOString(), + }, + }); }; } From fcb5b5f00a703ee59b2def4f98d362637e89bbd2 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 27 Mar 2024 20:42:35 +0100 Subject: [PATCH 03/31] rename aiops-api dir to aiops_api --- package.json | 4 ++-- tsconfig.base.json | 4 ++-- x-pack/plugins/{aiops-api => aiops_api}/README.md | 0 x-pack/plugins/{aiops-api => aiops_api}/jest.config.js | 6 +++--- x-pack/plugins/{aiops-api => aiops_api}/kibana.jsonc | 0 x-pack/plugins/{aiops-api => aiops_api}/server/index.ts | 0 .../plugins/{aiops-api => aiops_api}/server/lib/license.ts | 0 x-pack/plugins/{aiops-api => aiops_api}/server/plugin.ts | 0 .../server/routes/log_rate_analysis/define_route.ts | 0 .../routes/log_rate_analysis/route_handler_factory.ts | 0 x-pack/plugins/{aiops-api => aiops_api}/server/types.ts | 0 x-pack/plugins/{aiops-api => aiops_api}/tsconfig.json | 0 yarn.lock | 2 +- 13 files changed, 8 insertions(+), 8 deletions(-) rename x-pack/plugins/{aiops-api => aiops_api}/README.md (100%) rename x-pack/plugins/{aiops-api => aiops_api}/jest.config.js (76%) rename x-pack/plugins/{aiops-api => aiops_api}/kibana.jsonc (100%) rename x-pack/plugins/{aiops-api => aiops_api}/server/index.ts (100%) rename x-pack/plugins/{aiops-api => aiops_api}/server/lib/license.ts (100%) rename x-pack/plugins/{aiops-api => aiops_api}/server/plugin.ts (100%) rename x-pack/plugins/{aiops-api => aiops_api}/server/routes/log_rate_analysis/define_route.ts (100%) rename x-pack/plugins/{aiops-api => aiops_api}/server/routes/log_rate_analysis/route_handler_factory.ts (100%) rename x-pack/plugins/{aiops-api => aiops_api}/server/types.ts (100%) rename x-pack/plugins/{aiops-api => aiops_api}/tsconfig.json (100%) diff --git a/package.json b/package.json index 1acaf2d551e9b4..d3271bbb6c3203 100644 --- a/package.json +++ b/package.json @@ -148,7 +148,7 @@ "@kbn/actions-types": "link:packages/kbn-actions-types", "@kbn/advanced-settings-plugin": "link:src/plugins/advanced_settings", "@kbn/ai-assistant-management-plugin": "link:src/plugins/ai_assistant_management/selection", - "@kbn/aiops-api-plugin": "link:x-pack/plugins/aiops-api", + "@kbn/aiops-api-plugin": "link:x-pack/plugins/aiops_api", "@kbn/aiops-change-point-detection": "link:x-pack/packages/ml/aiops_change_point_detection", "@kbn/aiops-common": "link:x-pack/packages/ml/aiops_common", "@kbn/aiops-components": "link:x-pack/packages/ml/aiops_components", @@ -1760,4 +1760,4 @@ "zod-to-json-schema": "^3.22.3" }, "packageManager": "yarn@1.22.21" -} \ No newline at end of file +} diff --git a/tsconfig.base.json b/tsconfig.base.json index 1e0bf30c5fa868..bc0791ba9c5e43 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -18,8 +18,8 @@ "@kbn/advanced-settings-plugin/*": ["src/plugins/advanced_settings/*"], "@kbn/ai-assistant-management-plugin": ["src/plugins/ai_assistant_management/selection"], "@kbn/ai-assistant-management-plugin/*": ["src/plugins/ai_assistant_management/selection/*"], - "@kbn/aiops-api-plugin": ["x-pack/plugins/aiops-api"], - "@kbn/aiops-api-plugin/*": ["x-pack/plugins/aiops-api/*"], + "@kbn/aiops-api-plugin": ["x-pack/plugins/aiops_api"], + "@kbn/aiops-api-plugin/*": ["x-pack/plugins/aiops_api/*"], "@kbn/aiops-change-point-detection": ["x-pack/packages/ml/aiops_change_point_detection"], "@kbn/aiops-change-point-detection/*": ["x-pack/packages/ml/aiops_change_point_detection/*"], "@kbn/aiops-common": ["x-pack/packages/ml/aiops_common"], diff --git a/x-pack/plugins/aiops-api/README.md b/x-pack/plugins/aiops_api/README.md similarity index 100% rename from x-pack/plugins/aiops-api/README.md rename to x-pack/plugins/aiops_api/README.md diff --git a/x-pack/plugins/aiops-api/jest.config.js b/x-pack/plugins/aiops_api/jest.config.js similarity index 76% rename from x-pack/plugins/aiops-api/jest.config.js rename to x-pack/plugins/aiops_api/jest.config.js index 3c066ce0cd1d73..fec51ff18afbc8 100644 --- a/x-pack/plugins/aiops-api/jest.config.js +++ b/x-pack/plugins/aiops_api/jest.config.js @@ -8,8 +8,8 @@ module.exports = { preset: '@kbn/test', rootDir: '../../..', - roots: ['/x-pack/plugins/aiops-api'], - coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/aiops-api', + roots: ['/x-pack/plugins/aiops_api'], + coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/aiops_api', coverageReporters: ['text', 'html'], - collectCoverageFrom: ['/x-pack/plugins/aiops-api/server/**/*.{js,ts,tsx}'], + collectCoverageFrom: ['/x-pack/plugins/aiops_api/server/**/*.{js,ts,tsx}'], }; diff --git a/x-pack/plugins/aiops-api/kibana.jsonc b/x-pack/plugins/aiops_api/kibana.jsonc similarity index 100% rename from x-pack/plugins/aiops-api/kibana.jsonc rename to x-pack/plugins/aiops_api/kibana.jsonc diff --git a/x-pack/plugins/aiops-api/server/index.ts b/x-pack/plugins/aiops_api/server/index.ts similarity index 100% rename from x-pack/plugins/aiops-api/server/index.ts rename to x-pack/plugins/aiops_api/server/index.ts diff --git a/x-pack/plugins/aiops-api/server/lib/license.ts b/x-pack/plugins/aiops_api/server/lib/license.ts similarity index 100% rename from x-pack/plugins/aiops-api/server/lib/license.ts rename to x-pack/plugins/aiops_api/server/lib/license.ts diff --git a/x-pack/plugins/aiops-api/server/plugin.ts b/x-pack/plugins/aiops_api/server/plugin.ts similarity index 100% rename from x-pack/plugins/aiops-api/server/plugin.ts rename to x-pack/plugins/aiops_api/server/plugin.ts diff --git a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts similarity index 100% rename from x-pack/plugins/aiops-api/server/routes/log_rate_analysis/define_route.ts rename to x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts diff --git a/x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts similarity index 100% rename from x-pack/plugins/aiops-api/server/routes/log_rate_analysis/route_handler_factory.ts rename to x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts diff --git a/x-pack/plugins/aiops-api/server/types.ts b/x-pack/plugins/aiops_api/server/types.ts similarity index 100% rename from x-pack/plugins/aiops-api/server/types.ts rename to x-pack/plugins/aiops_api/server/types.ts diff --git a/x-pack/plugins/aiops-api/tsconfig.json b/x-pack/plugins/aiops_api/tsconfig.json similarity index 100% rename from x-pack/plugins/aiops-api/tsconfig.json rename to x-pack/plugins/aiops_api/tsconfig.json diff --git a/yarn.lock b/yarn.lock index cd630f959981a9..fe3b55106958f6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3249,7 +3249,7 @@ version "0.0.0" uid "" -"@kbn/aiops-api-plugin@link:x-pack/plugins/aiops-api": +"@kbn/aiops-api-plugin@link:x-pack/plugins/aiops_api": version "0.0.0" uid "" From 0d44ef14802d356632c01ee965a48010828214bf Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Thu, 28 Mar 2024 15:32:27 +0100 Subject: [PATCH 04/31] Obs AI Assistant <-> AIOps Log Rate Analysis --- x-pack/plugins/aiops_api/kibana.jsonc | 1 + .../get_aiops_log_rate_analysis_function.ts | 79 ++ .../server/assistant_functions/index.ts | 95 +++ x-pack/plugins/aiops_api/server/plugin.ts | 21 +- .../route_handler_factory.ts | 10 +- .../server/significant_terms_kibana_logs.ts | 167 +++++ .../server/significant_terms_pgbench.ts | 706 ++++++++++++++++++ x-pack/plugins/aiops_api/server/types.ts | 6 + 8 files changed, 1077 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts create mode 100644 x-pack/plugins/aiops_api/server/assistant_functions/index.ts create mode 100644 x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts create mode 100644 x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts diff --git a/x-pack/plugins/aiops_api/kibana.jsonc b/x-pack/plugins/aiops_api/kibana.jsonc index e5a3ba841ba90a..bcebc78072cbc0 100644 --- a/x-pack/plugins/aiops_api/kibana.jsonc +++ b/x-pack/plugins/aiops_api/kibana.jsonc @@ -9,6 +9,7 @@ "browser": false, "requiredPlugins": [ "licensing", + "observabilityAIAssistant" ], "optionalPlugins": [], "requiredBundles": [] diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts new file mode 100644 index 00000000000000..a843f35fa3b8b7 --- /dev/null +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { i18n } from '@kbn/i18n'; +import { FromSchema } from 'json-schema-to-ts'; +import { FunctionRegistrationParameters } from '.'; +// import { ApmTimeseries, getApmTimeseries } from '../routes/assistant_functions/get_apm_timeseries'; + +import { significantTerms } from '../significant_terms_kibana_logs'; + +export const NON_EMPTY_STRING = { + type: 'string' as const, + minLength: 1, +}; + +const parameters = { + type: 'object', + properties: { + start: { + type: 'string', + description: 'The start of the time range, in Elasticsearch date math, like `now`.', + }, + end: { + type: 'string', + description: 'The end of the time range, in Elasticsearch date math, like `now-24h`.', + }, + }, + required: [], +} as const; + +export function registerGetAiopsLogRateAnalysisFunction({ + apmEventClient, + registerFunction, +}: FunctionRegistrationParameters) { + registerFunction( + { + contexts: ['aiops'], + name: 'get_aiops_log_rate_analysis', + descriptionForUser: i18n.translate( + 'xpack.aiops.observabilityAiAssistant.functions.registerGetAiopsLogRateAnalyssi.descriptionForUser', + { + defaultMessage: `Log rate analysis is a feature that uses advanced statistical methods to identify reasons for increases or decreases in log rates.`, + } + ), + description: `Analyse a time series to identify log rate changes with contributing field/value pairs. The API returns significant field/value pairs found in the log rate change. The importance of field/value pairs is logIncrease descending. Briefly explain the data with value examples. Values with the same increase might correlate. Suggest actionable insights for remediations. Identify problematic users and services.`, + parameters, + }, + async ({ arguments: args }, signal): Promise => { + console.log('args', args); + return { + content: significantTerms + .filter(({ bg_count, doc_count }) => { + return doc_count > bg_count; + }) + .map(({ fieldName, fieldValue, type, doc_count, bg_count }) => ({ + field: fieldName, + value: fieldValue, + type: type === 'keyword' ? 'metadata' : 'log message pattern', + documentCount: doc_count, + baselineCount: bg_count, + logIncrease: + bg_count > 0 + ? `${Math.round((doc_count / bg_count) * 100) / 100}x increase` + : `${doc_count} documents up from 0 documents in baseline`, + })), + data: significantTerms, + }; + } + ); +} + +export type GetAiopsLogRateAnalysisFunctionArguments = FromSchema; +export interface GetAiopsLogRateAnalysisFunctionResponse { + content: Pick; + data: typeof significantTerms; +} diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/index.ts b/x-pack/plugins/aiops_api/server/assistant_functions/index.ts new file mode 100644 index 00000000000000..acae898cfedb6c --- /dev/null +++ b/x-pack/plugins/aiops_api/server/assistant_functions/index.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreSetup } from '@kbn/core-lifecycle-server'; +import type { Logger } from '@kbn/logging'; +import type { + RegistrationCallback, + RegisterFunction, +} from '@kbn/observability-ai-assistant-plugin/server/service/types'; +import type { IRuleDataClient } from '@kbn/rule-registry-plugin/server'; +// import type { APMConfig } from '..'; +// import type { ApmFeatureFlags } from '../../common/apm_feature_flags'; +// import { APMEventClient } from '../lib/helpers/create_es_client/create_apm_event_client'; +// import { getApmEventClient } from '../lib/helpers/get_apm_event_client'; +// import type { APMRouteHandlerResources } from '../routes/apm_routes/register_apm_server_routes'; +// import { hasHistoricalAgentData } from '../routes/historical_data/has_historical_agent_data'; +import { registerGetAiopsLogRateAnalysisFunction } from './get_aiops_log_rate_analysis_function'; + +export interface FunctionRegistrationParameters { + apmEventClient: any; // APMEventClient; + registerFunction: RegisterFunction; + resources: any; // APMRouteHandlerResources; +} + +export function registerAssistantFunctions({ + coreSetup, + config, + featureFlags, + logger, + kibanaVersion, + ruleDataClient, + plugins, +}: { + coreSetup: CoreSetup; + config: any; // APMConfig; + featureFlags: any; // ApmFeatureFlags; + logger: Logger; + kibanaVersion: string; + ruleDataClient: IRuleDataClient; + plugins: any; // APMRouteHandlerResources['plugins']; +}): RegistrationCallback { + return async ({ resources, functions: { registerContext, registerFunction } }) => { + // const apmRouteHandlerResources: APMRouteHandlerResources = { + // context: resources.context, + // request: resources.request, + // core: { + // setup: coreSetup, + // start: () => coreSetup.getStartServices().then(([coreStart]) => coreStart), + // }, + // params: { + // query: { + // _inspect: false, + // }, + // }, + // config, + // featureFlags, + // logger, + // kibanaVersion, + // ruleDataClient, + // plugins, + // getApmIndices: async () => { + // const coreContext = await resources.context.core; + // const apmIndices = await plugins.apmDataAccess.setup.getApmIndices( + // coreContext.savedObjects.client + // ); + // return apmIndices; + // }, + // }; + + // const apmEventClient = await getApmEventClient(apmRouteHandlerResources); + + // const hasData = await hasHistoricalAgentData(apmEventClient); + + // if (!hasData) { + // return; + // } + + const parameters: FunctionRegistrationParameters = { + resources: {}, + apmEventClient: {}, + registerFunction, + }; + + registerGetAiopsLogRateAnalysisFunction(parameters); + + registerContext({ + name: 'aiops', + description: ``, + }); + }; +} diff --git a/x-pack/plugins/aiops_api/server/plugin.ts b/x-pack/plugins/aiops_api/server/plugin.ts index 1f1842cdcf43e0..6ed9f80df8f61f 100755 --- a/x-pack/plugins/aiops_api/server/plugin.ts +++ b/x-pack/plugins/aiops_api/server/plugin.ts @@ -26,6 +26,7 @@ import type { AiopsApiPluginStartDeps, } from './types'; import { defineRoute as defineLogRateAnalysisRoute } from './routes/log_rate_analysis/define_route'; +import { registerAssistantFunctions } from './assistant_functions'; export class AiopsApiPlugin implements @@ -40,8 +41,9 @@ export class AiopsApiPlugin private licenseSubscription: Subscription | null = null; private usageCounter?: UsageCounter; - constructor(initializerContext: PluginInitializerContext) { - this.logger = initializerContext.logger.get(); + constructor(private readonly initContext: PluginInitializerContext) { + this.initContext = initContext; + this.logger = initContext.logger.get(); } public setup( @@ -66,6 +68,21 @@ export class AiopsApiPlugin defineLogRateAnalysisRoute(router, aiopsLicense, this.logger, coreStart, this.usageCounter); }); + // Register Observability AI Assistant functions + const kibanaVersion = this.initContext.env.packageInfo.version; + + plugins.observabilityAIAssistant.service.register( + registerAssistantFunctions({ + config: undefined, + coreSetup: core, + featureFlags: true, + kibanaVersion, + logger: this.logger.get('assistant'), + plugins: [], // resourcePlugins, + ruleDataClient: undefined, + }) + ); + return {}; } diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts index 55850385deb1a1..f0a1b480f1d5e0 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -26,10 +26,10 @@ import type { // import { trackAIOpsRouteUsage } from '../../lib/track_route_usage'; import type { AiopsApiLicense } from '../../types'; +import { significantTerms } from '../../significant_terms_kibana_logs'; + /** - * The log rate analysis route handler sets up `responseStreamFactory` - * to create the response stream and then uses its handlers to - * walk through the steps of the analysis. + * Log rate analysis route handler. */ export function routeHandlerFactory( version: T, @@ -59,9 +59,7 @@ export function routeHandlerFactory( // const executionContext = createExecutionContext(coreStart, AIOPS_PLUGIN_ID, request.route.path); return response.ok({ - body: { - time: new Date().toISOString(), - }, + body: significantTerms, }); }; } diff --git a/x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts b/x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts new file mode 100644 index 00000000000000..7437dc65844ddc --- /dev/null +++ b/x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const significantTerms = [ + { + key: 'GET /cgi-bin/mj_wwwusr passw list GLOBAL user func help extra /../../../../../../../../etc/passwd HTTP/1.1 Mozilla/5.0 X11 Linux i686 AppleWebKit/534.24 KHTML like Gecko Chrome/11.0.696.50 Safari/534.24', + fieldName: 'message', + fieldValue: + '30.156.16.163 - - [2018-09-01T12:43:51.756Z] "GET /cgi-bin/mj_wwwusr?passw=&list=GLOBAL&user=&func=help&extra=/../../../../../../../../etc/passwd HTTP/1.1" 404 1831 "-" "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"', + doc_count: 2, + bg_count: 0, + total_doc_count: 332, + total_bg_count: 3853, + score: -13.815510557964274, + pValue: 0.000001, + normalizedScore: 0, + type: 'log_pattern', + }, + { + key: 'response.keyword:404', + type: 'keyword', + fieldName: 'response.keyword', + fieldValue: '404', + doc_count: 110, + bg_count: 191, + total_doc_count: 329, + total_bg_count: 3853, + score: 79.84727915253413, + pValue: 2.1026513631459468e-35, + normalizedScore: 0.9129372030334755, + }, + { + key: 'geo.dest:IN', + type: 'keyword', + fieldName: 'geo.dest', + fieldValue: 'IN', + doc_count: 135, + bg_count: 590, + total_doc_count: 329, + total_bg_count: 3853, + score: 46.23195732860446, + pValue: 8.350568532508764e-21, + normalizedScore: 0.8299888399643792, + }, + { + key: 'agent.keyword:Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24', + type: 'keyword', + fieldName: 'agent.keyword', + fieldValue: + 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24', + doc_count: 179, + bg_count: 1249, + total_doc_count: 329, + total_bg_count: 3853, + score: 25.869244805538685, + pValue: 5.822771262698321e-12, + normalizedScore: 0.7797422982152977, + }, + { + key: 'machine.os.keyword:win xp', + type: 'keyword', + fieldName: 'machine.os.keyword', + fieldValue: 'win xp', + doc_count: 148, + bg_count: 765, + total_doc_count: 329, + total_bg_count: 3853, + score: 39.99917118025779, + pValue: 4.251876834760331e-18, + normalizedScore: 0.8146089661356224, + }, + { + key: 'tags.keyword:info', + type: 'keyword', + fieldName: 'tags.keyword', + fieldValue: 'info', + doc_count: 279, + bg_count: 2893, + total_doc_count: 329, + total_bg_count: 3853, + score: 4.462071350387152, + pValue: 0.011538438368795562, + normalizedScore: 0.09183161108299702, + }, + { + key: 'extension.keyword:', + type: 'keyword', + fieldName: 'extension.keyword', + fieldValue: '', + doc_count: 196, + bg_count: 1434, + total_doc_count: 329, + total_bg_count: 3853, + score: 25.887572445059007, + pValue: 5.717025604143256e-12, + normalizedScore: 0.7797875230596438, + }, + { + key: 'host.keyword:elastic-elastic-elastic.org', + type: 'keyword', + fieldName: 'host.keyword', + fieldValue: 'elastic-elastic-elastic.org', + doc_count: 112, + bg_count: 116, + total_doc_count: 329, + total_bg_count: 3853, + score: 102.24405969736205, + pValue: 3.94429206649694e-45, + normalizedScore: 0.9682029623185395, + }, + { + key: 'referer:http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', + type: 'keyword', + fieldName: 'referer', + fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', + doc_count: 101, + bg_count: 3, + total_doc_count: 329, + total_bg_count: 3853, + score: 121.69338765596726, + pValue: 1.410045922088992e-53, + normalizedScore: 1, + }, + { + key: 'geo.srcdest:US:IN', + type: 'keyword', + fieldName: 'geo.srcdest', + fieldValue: 'US:IN', + doc_count: 135, + bg_count: 590, + total_doc_count: 329, + total_bg_count: 3853, + score: 46.23195732860446, + pValue: 8.350568532508764e-21, + normalizedScore: 0.8299888399643792, + }, + { + key: 'clientip:30.156.16.164', + type: 'keyword', + fieldName: 'clientip', + fieldValue: '30.156.16.164', + doc_count: 100, + bg_count: 0, + total_doc_count: 329, + total_bg_count: 3853, + score: 121.00348032992358, + pValue: 2.8109699416779713e-53, + normalizedScore: 1, + }, + { + key: 'ip:30.156.16.163', + type: 'keyword', + fieldName: 'ip', + fieldValue: '30.156.16.163', + doc_count: 101, + bg_count: 2, + total_doc_count: 329, + total_bg_count: 3853, + score: 122.08801216439497, + pValue: 9.502765459867559e-54, + normalizedScore: 1, + }, +]; diff --git a/x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts b/x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts new file mode 100644 index 00000000000000..a40c78a60d54dd --- /dev/null +++ b/x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts @@ -0,0 +1,706 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const significantTerms = [ + { + key: 'postgres@mydb LOG duration ms statement UPDATE SET WHERE', + fieldName: 'message', + fieldValue: + '2022-06-07 07:24:07.725 GMT [619161] postgres@mydb LOG: duration: 4.210 ms statement: UPDATE pgbench_accounts SET abalance = abalance + -4868 WHERE aid = 141258;', + doc_count: 65618, + bg_count: 0, + total_doc_count: 315172, + total_bg_count: 223766, + score: -13.815510557964274, + pValue: 0.000001, + normalizedScore: 0, + type: 'log_pattern', + }, + { + key: 'postgres@mydb LOG duration ms statement', + fieldName: 'message', + fieldValue: + '2022-06-07 07:23:12.506 GMT [619058] postgres@mydb LOG: duration: 0.816 ms statement: begin', + doc_count: 153675, + bg_count: 0, + total_doc_count: 315172, + total_bg_count: 223766, + score: -13.815510557964274, + pValue: 0.000001, + normalizedScore: 0, + type: 'log_pattern', + }, + { + key: 'postgres@mydb LOG duration ms statement INSERT INTO pgbench_history tid bid aid delta mtime VALUES CURRENT_TIMESTAMP', + fieldName: 'message', + fieldValue: + '2022-06-07 07:24:07.786 GMT [619164] postgres@mydb LOG: duration: 0.207 ms statement: INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (204, 41, 2624135, -2382, CURRENT_TIMESTAMP);', + doc_count: 21873, + bg_count: 0, + total_doc_count: 315172, + total_bg_count: 223766, + score: -13.815510557964274, + pValue: 0.000001, + normalizedScore: 0, + type: 'log_pattern', + }, + { + key: 'postgres@mydb LOG duration ms statement insert into bid values', + fieldName: 'message', + fieldValue: + '2022-06-07 07:23:12.509 GMT [619058] postgres@mydb LOG: duration: 0.057 ms statement: insert into pgbench_branches(bid,bbalance) values(10,0)', + doc_count: 22423, + bg_count: 0, + total_doc_count: 315172, + total_bg_count: 223766, + score: -13.815510557964274, + pValue: 0.000001, + normalizedScore: 0, + type: 'log_pattern', + }, + { + key: 'kubernetes.node.uid:641a2bef-4ea2-47c2-92f7-379ea1137e26', + type: 'keyword', + fieldName: 'kubernetes.node.uid', + fieldValue: '641a2bef-4ea2-47c2-92f7-379ea1137e26', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'host.mac:0e:56:50:16:32:5c', + type: 'keyword', + fieldName: 'host.mac', + fieldValue: '0e:56:50:16:32:5c', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'host.hostname:filebeat-t8pfz', + type: 'keyword', + fieldName: 'host.hostname', + fieldValue: 'filebeat-t8pfz', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.id:76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156', + type: 'keyword', + fieldName: 'container.id', + fieldValue: '76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'service.type:postgresql', + type: 'keyword', + fieldName: 'service.type', + fieldValue: 'postgresql', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'fileset.name:log', + type: 'keyword', + fieldName: 'fileset.name', + fieldValue: 'log', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'agent.hostname:filebeat-t8pfz', + type: 'keyword', + fieldName: 'agent.hostname', + fieldValue: 'filebeat-t8pfz', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'host.ip:10.40.5.185', + type: 'keyword', + fieldName: 'host.ip', + fieldValue: '10.40.5.185', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.node.hostname:gke-eden-3-staging-ssd-3-bc7684a4-2isy.c.elastic-product.internal', + type: 'keyword', + fieldName: 'kubernetes.node.hostname', + fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy.c.elastic-product.internal', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.pod.ip:10.40.5.6', + type: 'keyword', + fieldName: 'kubernetes.pod.ip', + fieldValue: '10.40.5.6', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'postgresql.log.database:pgbench', + type: 'keyword', + fieldName: 'postgresql.log.database', + fieldValue: 'pgbench', + doc_count: 150443, + bg_count: 0, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.container.name:postgres', + type: 'keyword', + fieldName: 'kubernetes.container.name', + fieldValue: 'postgres', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'log.file.path:/var/lib/docker/containers/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156-json.log', + type: 'keyword', + fieldName: 'log.file.path', + fieldValue: + '/var/lib/docker/containers/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156-json.log', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'cloud.instance.name:gke-eden-3-staging-ssd-3-bc7684a4-2isy', + type: 'keyword', + fieldName: 'cloud.instance.name', + fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.labels.io_kubernetes_container_name:postgres', + type: 'keyword', + fieldName: 'container.labels.io_kubernetes_container_name', + fieldValue: 'postgres', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.labels.pod-template-hash:5dc4ff9cf7', + type: 'keyword', + fieldName: 'kubernetes.labels.pod-template-hash', + fieldValue: '5dc4ff9cf7', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'log.level:LOG', + type: 'keyword', + fieldName: 'log.level', + fieldValue: 'LOG', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'postgresql.log.query:BEGIN;', + type: 'keyword', + fieldName: 'postgresql.log.query', + fieldValue: 'BEGIN;', + doc_count: 22214, + bg_count: 0, + total_doc_count: 178871, + total_bg_count: 221943, + score: 545.6888415128909, + pValue: 1.02411162216583e-237, + normalizedScore: 1, + }, + { + key: 'postgresql.log.query:END;', + type: 'keyword', + fieldName: 'postgresql.log.query', + fieldValue: 'END;', + doc_count: 20757, + bg_count: 0, + total_doc_count: 178871, + total_bg_count: 221943, + score: 489.1397423802259, + pValue: 3.709445585218203e-213, + normalizedScore: 1, + }, + { + key: 'event.module:postgresql', + type: 'keyword', + fieldName: 'event.module', + fieldValue: 'postgresql', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'service.name:postgres', + type: 'keyword', + fieldName: 'service.name', + fieldValue: 'postgres', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.pod.name:postgres-5dc4ff9cf7-flc4h', + type: 'keyword', + fieldName: 'kubernetes.pod.name', + fieldValue: 'postgres-5dc4ff9cf7-flc4h', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'event.timezone:GMT', + type: 'keyword', + fieldName: 'event.timezone', + fieldValue: 'GMT', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.node.labels.kubernetes_io/hostname:gke-eden-3-staging-ssd-3-bc7684a4-2isy', + type: 'keyword', + fieldName: 'kubernetes.node.labels.kubernetes_io/hostname', + fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.labels.app:postgres', + type: 'keyword', + fieldName: 'kubernetes.labels.app', + fieldValue: 'postgres', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'agent.name:filebeat-t8pfz', + type: 'keyword', + fieldName: 'agent.name', + fieldValue: 'filebeat-t8pfz', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'event.category:database', + type: 'keyword', + fieldName: 'event.category', + fieldValue: 'database', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.image.name:docker.elastic.co/sa/observe-postgres-hipster@sha256:d6e54ed8180612c98cfd7ea6fab35c5263c5bd7845d2d95d81af78e21febd6b4', + type: 'keyword', + fieldName: 'container.image.name', + fieldValue: + 'docker.elastic.co/sa/observe-postgres-hipster@sha256:d6e54ed8180612c98cfd7ea6fab35c5263c5bd7845d2d95d81af78e21febd6b4', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.labels.io_kubernetes_pod_uid:0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', + type: 'keyword', + fieldName: 'container.labels.io_kubernetes_pod_uid', + fieldValue: '0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'related.user:pgbench', + type: 'keyword', + fieldName: 'related.user', + fieldValue: 'pgbench', + doc_count: 150443, + bg_count: 0, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'agent.id:8a7d20ae-da85-4b50-9c7c-2b93ab794216', + type: 'keyword', + fieldName: 'agent.id', + fieldValue: '8a7d20ae-da85-4b50-9c7c-2b93ab794216', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.labels.io_kubernetes_sandbox_id:a7caeb81df949a4da817fe93879dfbe31e337fe059c128ea36e9661f88883427', + type: 'keyword', + fieldName: 'container.labels.io_kubernetes_sandbox_id', + fieldValue: 'a7caeb81df949a4da817fe93879dfbe31e337fe059c128ea36e9661f88883427', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.labels.annotation_io_kubernetes_container_hash:55d6093a', + type: 'keyword', + fieldName: 'container.labels.annotation_io_kubernetes_container_hash', + fieldValue: '55d6093a', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'user.name:pgbench', + type: 'keyword', + fieldName: 'user.name', + fieldValue: 'pgbench', + doc_count: 150443, + bg_count: 0, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'postgresql.log.query_step:statement', + type: 'keyword', + fieldName: 'postgresql.log.query_step', + fieldValue: 'statement', + doc_count: 155100, + bg_count: 40186, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'cloud.instance.id:2037777022342535795', + type: 'keyword', + fieldName: 'cloud.instance.id', + fieldValue: '2037777022342535795', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'stream:stderr', + type: 'keyword', + fieldName: 'stream', + fieldValue: 'stderr', + doc_count: 166214, + bg_count: 127186, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.replicaset.name:postgres-5dc4ff9cf7', + type: 'keyword', + fieldName: 'kubernetes.replicaset.name', + fieldValue: 'postgres-5dc4ff9cf7', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'agent.ephemeral_id:b66cd0b0-05cf-4d5a-bda9-7b27d5e80e2b', + type: 'keyword', + fieldName: 'agent.ephemeral_id', + fieldValue: 'b66cd0b0-05cf-4d5a-bda9-7b27d5e80e2b', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'event.kind:event', + type: 'keyword', + fieldName: 'event.kind', + fieldValue: 'event', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.name:k8s_postgres_postgres-5dc4ff9cf7-flc4h_observe-shop_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4_0', + type: 'keyword', + fieldName: 'container.name', + fieldValue: + 'k8s_postgres_postgres-5dc4ff9cf7-flc4h_observe-shop_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4_0', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'host.name:gke-eden-3-staging-ssd-3-bc7684a4-2isy', + type: 'keyword', + fieldName: 'host.name', + fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'ecs.version:1.12.0', + type: 'keyword', + fieldName: 'ecs.version', + fieldValue: '1.12.0', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.labels.io_kubernetes_pod_name:postgres-5dc4ff9cf7-flc4h', + type: 'keyword', + fieldName: 'container.labels.io_kubernetes_pod_name', + fieldValue: 'postgres-5dc4ff9cf7-flc4h', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.pod.uid:0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', + type: 'keyword', + fieldName: 'kubernetes.pod.uid', + fieldValue: '0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'kubernetes.node.name:gke-eden-3-staging-ssd-3-bc7684a4-2isy', + type: 'keyword', + fieldName: 'kubernetes.node.name', + fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', + doc_count: 165757, + bg_count: 123157, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'event.type:info', + type: 'keyword', + fieldName: 'event.type', + fieldValue: 'info', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'container.labels.io_kubernetes_container_logpath:/var/log/pods/observe-shop_postgres-5dc4ff9cf7-flc4h_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4/postgres/0.log', + type: 'keyword', + fieldName: 'container.labels.io_kubernetes_container_logpath', + fieldValue: + '/var/log/pods/observe-shop_postgres-5dc4ff9cf7-flc4h_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4/postgres/0.log', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, + { + key: 'event.dataset:postgresql.log', + type: 'keyword', + fieldName: 'event.dataset', + fieldValue: 'postgresql.log', + doc_count: 157286, + bg_count: 59214, + total_doc_count: 178871, + total_bg_count: 221943, + score: 708.3964185322641, + pValue: 2.2250738585072626e-308, + normalizedScore: 1, + }, +]; diff --git a/x-pack/plugins/aiops_api/server/types.ts b/x-pack/plugins/aiops_api/server/types.ts index 00081e95d5374c..7dff5e466bfdfd 100755 --- a/x-pack/plugins/aiops_api/server/types.ts +++ b/x-pack/plugins/aiops_api/server/types.ts @@ -9,16 +9,22 @@ import type { PluginSetup, PluginStart } from '@kbn/data-plugin/server'; import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import type { CasesServerSetup } from '@kbn/cases-plugin/server'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; +import type { + ObservabilityAIAssistantServerSetup, + ObservabilityAIAssistantServerStart, +} from '@kbn/observability-ai-assistant-plugin/server'; export interface AiopsApiPluginSetupDeps { data: PluginSetup; licensing: LicensingPluginSetup; + observabilityAIAssistant: ObservabilityAIAssistantServerSetup; cases?: CasesServerSetup; usageCollection?: UsageCollectionSetup; } export interface AiopsApiPluginStartDeps { data: PluginStart; + observabilityAIAssistant: ObservabilityAIAssistantServerStart; } /** From f19074af1396a4e8a7583c4c39922323a7e811c7 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 29 Mar 2024 14:51:34 +0100 Subject: [PATCH 05/31] working PoC --- .../components/top_nav/discover_topnav.tsx | 13 + x-pack/packages/ml/local_storage/index.ts | 1 + .../src/use_local_storage_listener.ts | 63 ++ .../get_aiops_log_rate_analysis_function.ts | 292 +++++++- .../server/assistant_functions/index.ts | 4 +- .../route_handler_factory.ts | 4 +- .../server/significant_terms_kibana_logs.ts | 167 ----- .../server/significant_terms_pgbench.ts | 706 ------------------ .../public/components/nav_control/index.tsx | 6 +- .../hooks/use_nav_control_screen_context.ts | 12 +- 10 files changed, 364 insertions(+), 904 deletions(-) create mode 100644 x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts delete mode 100644 x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts delete mode 100644 x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts diff --git a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx index 55fe8825477d9f..a1509338248372 100644 --- a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx @@ -7,6 +7,7 @@ */ import React, { useCallback, useEffect, useMemo, useRef } from 'react'; +import { useLocalStorageListener } from '@kbn/ml-local-storage'; import { type DataView, DataViewType } from '@kbn/data-views-plugin/public'; import { DataViewPickerProps } from '@kbn/unified-search-plugin/public'; import { ENABLE_ESQL } from '@kbn/esql-utils'; @@ -71,6 +72,12 @@ export const DiscoverTopNav = ({ const closeFieldEditor = useRef<() => void | undefined>(); const closeDataViewEditor = useRef<() => void | undefined>(); + const [lsIndex, saveLsIndex] = useLocalStorageListener('obs-ai-assistant-index', null); + const [lsIndexTimeField, saveLsIndexTimeField] = useLocalStorageListener( + 'obs-ai-assistant-index-time-field', + null + ); + useEffect(() => { return () => { // Make sure to close the editors when unmounting @@ -181,6 +188,12 @@ export const DiscoverTopNav = ({ topNavMenu, ]); + useEffect(() => { + console.log('time field', dataView?.getTimeField()?.spec.name); + saveLsIndex(dataView?.getIndexPattern() || null); + saveLsIndexTimeField(dataView?.getTimeField()?.spec.name || null); + }, [dataView]); + const savedSearchId = useSavedSearch().id; const savedSearchHasChanged = useSavedSearchHasChanged(); const dataViewPickerProps: DataViewPickerProps = useMemo(() => { diff --git a/x-pack/packages/ml/local_storage/index.ts b/x-pack/packages/ml/local_storage/index.ts index f950f8791a3415..c4a41bebe328b4 100644 --- a/x-pack/packages/ml/local_storage/index.ts +++ b/x-pack/packages/ml/local_storage/index.ts @@ -6,3 +6,4 @@ */ export { StorageContextProvider, useStorage } from './src/storage_context'; +export { useLocalStorageListener } from './src/use_local_storage_listener'; diff --git a/x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts b/x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts new file mode 100644 index 00000000000000..3bf82ba4edf728 --- /dev/null +++ b/x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { useEffect, useState } from 'react'; + +const EVENT_NAME_PREFIX = 'local-storage-listener'; + +export function useLocalStorageListener(key: string, initValue: any) { + const [state, setState] = useState(() => { + const value = localStorage.getItem(key); + if (value !== null) { + try { + return JSON.parse(value); + } catch (e) { + return null; + } + } + + localStorage.setItem(key, JSON.stringify(initValue)); + window.dispatchEvent(new Event(`${EVENT_NAME_PREFIX}${key}`)); + return initValue; + }); + + useEffect(() => { + const value = localStorage.getItem(key); + if (value !== JSON.stringify(state)) { + localStorage.setItem(key, JSON.stringify(state)); + window.dispatchEvent(new Event(`${EVENT_NAME_PREFIX}${key}`)); + } + }, [key, state]); + + useEffect(() => { + const listenStorageChange = () => { + setState(() => { + const value = localStorage.getItem(key); + if (value !== null) { + try { + return JSON.parse(value); + } catch (e) { + return null; + } + } + + if (initValue !== null) { + localStorage.setItem(key, JSON.stringify(initValue)); + window.dispatchEvent(new Event(`${EVENT_NAME_PREFIX}${key}`)); + return initValue; + } + + return null; + }); + }; + window.addEventListener(`${EVENT_NAME_PREFIX}${key}`, listenStorageChange); + return () => window.removeEventListener(`${EVENT_NAME_PREFIX}${key}`, listenStorageChange); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return [state, setState]; +} diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index a843f35fa3b8b7..4d2aea1c91e6d6 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -4,12 +4,33 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { i18n } from '@kbn/i18n'; + +import { queue } from 'async'; import { FromSchema } from 'json-schema-to-ts'; +import { mean } from 'd3-array'; +import moment from 'moment'; + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { type SignificantItem } from '@kbn/ml-agg-utils'; +import { i18n } from '@kbn/i18n'; +import dateMath from '@kbn/datemath'; +import { + getExtendedChangePoint, + getWindowParametersForTrigger, +} from '@kbn/aiops-log-rate-analysis'; +import { fetchIndexInfo } from '@kbn/aiops-log-rate-analysis/queries/fetch_index_info'; +import type { AiopsLogRateAnalysisSchema } from '@kbn/aiops-log-rate-analysis/api/schema'; +import { fetchSignificantCategories } from '@kbn/aiops-log-rate-analysis/queries/fetch_significant_categories'; +import { fetchSignificantTermPValues } from '@kbn/aiops-log-rate-analysis/queries/fetch_significant_term_p_values'; +import { getSampleProbability } from '@kbn/ml-random-sampler-utils'; + import { FunctionRegistrationParameters } from '.'; // import { ApmTimeseries, getApmTimeseries } from '../routes/assistant_functions/get_apm_timeseries'; -import { significantTerms } from '../significant_terms_kibana_logs'; +// Don't use more than 10 here otherwise Kibana will emit an error +// regarding a limit of abort signal listeners of more than 10. +export const MAX_CONCURRENT_QUERIES = 5; export const NON_EMPTY_STRING = { type: 'string' as const, @@ -19,6 +40,14 @@ export const NON_EMPTY_STRING = { const parameters = { type: 'object', properties: { + index: { + type: 'string', + description: 'The Elasticsearch source index pattern.', + }, + timefield: { + type: 'string', + description: 'The Elasticesarch source index pattern time field.', + }, start: { type: 'string', description: 'The start of the time range, in Elasticsearch date math, like `now`.', @@ -28,11 +57,12 @@ const parameters = { description: 'The end of the time range, in Elasticsearch date math, like `now-24h`.', }, }, - required: [], + required: ['index', 'timefield', 'start', 'end'], } as const; export function registerGetAiopsLogRateAnalysisFunction({ apmEventClient, + resources, registerFunction, }: FunctionRegistrationParameters) { registerFunction( @@ -45,28 +75,246 @@ export function registerGetAiopsLogRateAnalysisFunction({ defaultMessage: `Log rate analysis is a feature that uses advanced statistical methods to identify reasons for increases or decreases in log rates.`, } ), - description: `Analyse a time series to identify log rate changes with contributing field/value pairs. The API returns significant field/value pairs found in the log rate change. The importance of field/value pairs is logIncrease descending. Briefly explain the data with value examples. Values with the same increase might correlate. Suggest actionable insights for remediations. Identify problematic users and services.`, + description: `Log rate analysis is a Elastic AIOps feature that uses advanced statistical methods to identify reasons for increases or decreases in time series of log rates. The analysis returns significant field/value pairs found in the log rate change. The importance of field/value pairs is logIncrease descending. Briefly explain the data with value examples. Values with the same increase might correlate. Suggest actionable insights for remediations and identify potential security and performance issues.`, parameters, }, - async ({ arguments: args }, signal): Promise => { + async ({ arguments: args }, abortSignal): Promise => { + const debugStartTime = Date.now(); + const { esClient } = resources; console.log('args', args); + + // CHANGE POINT DETECTION + + const barTarget = 70; + const earliestMs = dateMath.parse(args.start)?.valueOf(); + const latestMs = dateMath.parse(args.end, { roundUp: true })?.valueOf(); + + if (earliestMs === undefined || latestMs === undefined) { + return { content: 'Could not parse time range.', data: [] }; + } + const delta = latestMs - earliestMs; + const dayMs = 86400 * 1000; + const threshold = dayMs * 22; + const intervalMs = delta > threshold ? dayMs : Math.round(delta / barTarget); + + const aggs: Record = { + eventRate: { + date_histogram: { + field: args.timefield, + fixed_interval: `${intervalMs}ms`, + min_doc_count: 0, + ...(earliestMs !== undefined && latestMs !== undefined + ? { + extended_bounds: { + min: earliestMs, + max: latestMs, + }, + } + : {}), + }, + }, + change_point_request: { + // @ts-expect-error missing from ES spec + change_point: { + buckets_path: 'eventRate>_count', + }, + }, + }; + + const searchQuery = { + range: { + [args.timefield]: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, + }; + const searchBody: estypes.MsearchMultisearchBody = { + query: searchQuery, + aggs, + track_total_hits: false, + size: 0, + }; + + const histogram = await esClient.search( + { + index: args.index, + body: searchBody, + }, + { + // signal: abortSignal, + maxRetries: 0, + } + ); + + console.log('eventRate', histogram.aggregations.eventRate); + console.log('change_point_request', histogram.aggregations.change_point_request); + + if (histogram.aggregations.change_point_request.bucket === undefined) { + return { content: 'No log rate change detected.', data: [] }; + } + + const buckets = histogram.aggregations.eventRate.buckets.reduce((acc, cur) => { + acc[cur.key] = cur.doc_count; + return acc; + }, {}); + + const changePointTs = dateMath + .parse(histogram.aggregations.change_point_request.bucket.key) + ?.valueOf(); + + if (changePointTs === undefined) { + return { content: 'There was an error parsing the log rate change timestamp.', data: [] }; + } + + const extendedChangePoint = getExtendedChangePoint(buckets, changePointTs); + console.log('extendedChangePoint', extendedChangePoint); + const logRateType = Object.keys(histogram.aggregations.change_point_request.type)[0]; + const logRateAttributeName = `${logRateType}LogRate`; + + // FIELD CANDIDATES + + const fieldCandidates: string[] = []; + let fieldCandidatesCount = fieldCandidates.length; + const textFieldCandidates: string[] = []; + let totalDocCount = 0; + let zeroDocsFallback = false; + const changePoint = { + ...extendedChangePoint, + key: changePointTs, + type: logRateType, + }; + const wp = getWindowParametersForTrigger( + extendedChangePoint.startTs, + intervalMs, + earliestMs, + latestMs, + changePoint + ); + + const params: AiopsLogRateAnalysisSchema = { + index: args.index, + start: earliestMs, + end: latestMs, + searchQuery: JSON.stringify(searchQuery), + timeFieldName: args.timefield, + ...wp, + }; + + const indexInfo = await fetchIndexInfo( + esClient, + params, + ['message', 'error.message'], + abortSignal + ); + console.log('indexInfo', indexInfo); + + fieldCandidates.push(...indexInfo.fieldCandidates); + fieldCandidatesCount = fieldCandidates.length; + textFieldCandidates.push(...indexInfo.textFieldCandidates); + totalDocCount = indexInfo.deviationTotalDocCount; + zeroDocsFallback = indexInfo.zeroDocsFallback; + const sampleProbability = getSampleProbability( + indexInfo.deviationTotalDocCount + indexInfo.baselineTotalDocCount + ); + console.log('sampleProbability', sampleProbability); + + // SIGNIFICANT ITEMS + + fieldCandidatesCount = fieldCandidates.length; + + // This will store the combined count of detected significant log patterns and keywords + let fieldValuePairsCount = 0; + + const significantCategories: SignificantItem[] = []; + + // Get significant categories of text fields + if (textFieldCandidates.length > 100) { + significantCategories.push( + ...(await fetchSignificantCategories( + esClient, + params, + textFieldCandidates, + undefined, + sampleProbability, + () => {}, + abortSignal + )) + ); + } + + const significantTerms: SignificantItem[] = []; + const fieldsToSample = new Set(); + + const pValuesQueue = queue(async function (fieldCandidate: string) { + const pValues = await fetchSignificantTermPValues( + esClient, + params, + [fieldCandidate], + undefined, + sampleProbability, + () => {}, + abortSignal + ); + + if (pValues.length > 0) { + pValues.forEach((d) => { + fieldsToSample.add(d.fieldName); + }); + significantTerms.push(...pValues); + } + }, MAX_CONCURRENT_QUERIES); + + pValuesQueue.push(fieldCandidates, (err) => { + if (err) { + pValuesQueue.kill(); + } + }); + await pValuesQueue.drain(); + + fieldValuePairsCount = significantCategories.length + significantTerms.length; + + if (fieldValuePairsCount === 0) { + return { content: 'Log rate analysis did not identify any significant items.', data: [] }; + } + + // RETURN DATA + const debugEndTime = Date.now(); + const debugDelta = (debugEndTime - debugStartTime) / 1000; + console.log(`Took: ${debugDelta}s`); + return { - content: significantTerms - .filter(({ bg_count, doc_count }) => { - return doc_count > bg_count; - }) - .map(({ fieldName, fieldValue, type, doc_count, bg_count }) => ({ - field: fieldName, - value: fieldValue, - type: type === 'keyword' ? 'metadata' : 'log message pattern', - documentCount: doc_count, - baselineCount: bg_count, - logIncrease: - bg_count > 0 - ? `${Math.round((doc_count / bg_count) * 100) / 100}x increase` - : `${doc_count} documents up from 0 documents in baseline`, - })), - data: significantTerms, + content: { + logRateChange: { + type: Object.keys(histogram.aggregations.change_point_request.type)[0], + timestamp: histogram.aggregations.change_point_request.bucket.key, + [logRateAttributeName]: histogram.aggregations.change_point_request.bucket.doc_count, + averageLogRate: Math.round(mean(Object.values(buckets)) ?? 0), + logRateAggregationIntervalUsedForAnalysis: moment + .duration(Math.round(intervalMs / 1000), 'seconds') + .humanize(), + ...(sampleProbability < 1 + ? { documentSamplingFactorForAnalysis: sampleProbability } + : {}), + }, + significantItems: [...significantTerms, ...significantCategories] + .filter(({ bg_count, doc_count }) => { + return doc_count > bg_count; + }) + .map(({ fieldName, fieldValue, type, doc_count, bg_count }) => ({ + field: fieldName, + value: fieldValue, + type: type === 'keyword' ? 'metadata' : 'log message pattern', + documentCount: doc_count, + baselineCount: bg_count, + logIncrease: + bg_count > 0 + ? `${Math.round((doc_count / bg_count) * 100) / 100}x increase` + : `${doc_count} documents up from 0 documents in baseline`, + })), + }, + data: [...significantTerms, ...significantCategories], }; } ); @@ -74,6 +322,6 @@ export function registerGetAiopsLogRateAnalysisFunction({ export type GetAiopsLogRateAnalysisFunctionArguments = FromSchema; export interface GetAiopsLogRateAnalysisFunctionResponse { - content: Pick; + content: any; data: typeof significantTerms; } diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/index.ts b/x-pack/plugins/aiops_api/server/assistant_functions/index.ts index acae898cfedb6c..bb3ace05b4de0c 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/index.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/index.ts @@ -79,8 +79,10 @@ export function registerAssistantFunctions({ // return; // } + const esClient = await (await resources.context.core).elasticsearch.client.asCurrentUser; + const parameters: FunctionRegistrationParameters = { - resources: {}, + resources: { esClient }, apmEventClient: {}, registerFunction, }; diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts index f0a1b480f1d5e0..650d1379721ed7 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -26,8 +26,6 @@ import type { // import { trackAIOpsRouteUsage } from '../../lib/track_route_usage'; import type { AiopsApiLicense } from '../../types'; -import { significantTerms } from '../../significant_terms_kibana_logs'; - /** * Log rate analysis route handler. */ @@ -59,7 +57,7 @@ export function routeHandlerFactory( // const executionContext = createExecutionContext(coreStart, AIOPS_PLUGIN_ID, request.route.path); return response.ok({ - body: significantTerms, + body: [], }); }; } diff --git a/x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts b/x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts deleted file mode 100644 index 7437dc65844ddc..00000000000000 --- a/x-pack/plugins/aiops_api/server/significant_terms_kibana_logs.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const significantTerms = [ - { - key: 'GET /cgi-bin/mj_wwwusr passw list GLOBAL user func help extra /../../../../../../../../etc/passwd HTTP/1.1 Mozilla/5.0 X11 Linux i686 AppleWebKit/534.24 KHTML like Gecko Chrome/11.0.696.50 Safari/534.24', - fieldName: 'message', - fieldValue: - '30.156.16.163 - - [2018-09-01T12:43:51.756Z] "GET /cgi-bin/mj_wwwusr?passw=&list=GLOBAL&user=&func=help&extra=/../../../../../../../../etc/passwd HTTP/1.1" 404 1831 "-" "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24"', - doc_count: 2, - bg_count: 0, - total_doc_count: 332, - total_bg_count: 3853, - score: -13.815510557964274, - pValue: 0.000001, - normalizedScore: 0, - type: 'log_pattern', - }, - { - key: 'response.keyword:404', - type: 'keyword', - fieldName: 'response.keyword', - fieldValue: '404', - doc_count: 110, - bg_count: 191, - total_doc_count: 329, - total_bg_count: 3853, - score: 79.84727915253413, - pValue: 2.1026513631459468e-35, - normalizedScore: 0.9129372030334755, - }, - { - key: 'geo.dest:IN', - type: 'keyword', - fieldName: 'geo.dest', - fieldValue: 'IN', - doc_count: 135, - bg_count: 590, - total_doc_count: 329, - total_bg_count: 3853, - score: 46.23195732860446, - pValue: 8.350568532508764e-21, - normalizedScore: 0.8299888399643792, - }, - { - key: 'agent.keyword:Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24', - type: 'keyword', - fieldName: 'agent.keyword', - fieldValue: - 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24', - doc_count: 179, - bg_count: 1249, - total_doc_count: 329, - total_bg_count: 3853, - score: 25.869244805538685, - pValue: 5.822771262698321e-12, - normalizedScore: 0.7797422982152977, - }, - { - key: 'machine.os.keyword:win xp', - type: 'keyword', - fieldName: 'machine.os.keyword', - fieldValue: 'win xp', - doc_count: 148, - bg_count: 765, - total_doc_count: 329, - total_bg_count: 3853, - score: 39.99917118025779, - pValue: 4.251876834760331e-18, - normalizedScore: 0.8146089661356224, - }, - { - key: 'tags.keyword:info', - type: 'keyword', - fieldName: 'tags.keyword', - fieldValue: 'info', - doc_count: 279, - bg_count: 2893, - total_doc_count: 329, - total_bg_count: 3853, - score: 4.462071350387152, - pValue: 0.011538438368795562, - normalizedScore: 0.09183161108299702, - }, - { - key: 'extension.keyword:', - type: 'keyword', - fieldName: 'extension.keyword', - fieldValue: '', - doc_count: 196, - bg_count: 1434, - total_doc_count: 329, - total_bg_count: 3853, - score: 25.887572445059007, - pValue: 5.717025604143256e-12, - normalizedScore: 0.7797875230596438, - }, - { - key: 'host.keyword:elastic-elastic-elastic.org', - type: 'keyword', - fieldName: 'host.keyword', - fieldValue: 'elastic-elastic-elastic.org', - doc_count: 112, - bg_count: 116, - total_doc_count: 329, - total_bg_count: 3853, - score: 102.24405969736205, - pValue: 3.94429206649694e-45, - normalizedScore: 0.9682029623185395, - }, - { - key: 'referer:http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', - type: 'keyword', - fieldName: 'referer', - fieldValue: 'http://www.elastic-elastic-elastic.com/success/timothy-l-kopra', - doc_count: 101, - bg_count: 3, - total_doc_count: 329, - total_bg_count: 3853, - score: 121.69338765596726, - pValue: 1.410045922088992e-53, - normalizedScore: 1, - }, - { - key: 'geo.srcdest:US:IN', - type: 'keyword', - fieldName: 'geo.srcdest', - fieldValue: 'US:IN', - doc_count: 135, - bg_count: 590, - total_doc_count: 329, - total_bg_count: 3853, - score: 46.23195732860446, - pValue: 8.350568532508764e-21, - normalizedScore: 0.8299888399643792, - }, - { - key: 'clientip:30.156.16.164', - type: 'keyword', - fieldName: 'clientip', - fieldValue: '30.156.16.164', - doc_count: 100, - bg_count: 0, - total_doc_count: 329, - total_bg_count: 3853, - score: 121.00348032992358, - pValue: 2.8109699416779713e-53, - normalizedScore: 1, - }, - { - key: 'ip:30.156.16.163', - type: 'keyword', - fieldName: 'ip', - fieldValue: '30.156.16.163', - doc_count: 101, - bg_count: 2, - total_doc_count: 329, - total_bg_count: 3853, - score: 122.08801216439497, - pValue: 9.502765459867559e-54, - normalizedScore: 1, - }, -]; diff --git a/x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts b/x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts deleted file mode 100644 index a40c78a60d54dd..00000000000000 --- a/x-pack/plugins/aiops_api/server/significant_terms_pgbench.ts +++ /dev/null @@ -1,706 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const significantTerms = [ - { - key: 'postgres@mydb LOG duration ms statement UPDATE SET WHERE', - fieldName: 'message', - fieldValue: - '2022-06-07 07:24:07.725 GMT [619161] postgres@mydb LOG: duration: 4.210 ms statement: UPDATE pgbench_accounts SET abalance = abalance + -4868 WHERE aid = 141258;', - doc_count: 65618, - bg_count: 0, - total_doc_count: 315172, - total_bg_count: 223766, - score: -13.815510557964274, - pValue: 0.000001, - normalizedScore: 0, - type: 'log_pattern', - }, - { - key: 'postgres@mydb LOG duration ms statement', - fieldName: 'message', - fieldValue: - '2022-06-07 07:23:12.506 GMT [619058] postgres@mydb LOG: duration: 0.816 ms statement: begin', - doc_count: 153675, - bg_count: 0, - total_doc_count: 315172, - total_bg_count: 223766, - score: -13.815510557964274, - pValue: 0.000001, - normalizedScore: 0, - type: 'log_pattern', - }, - { - key: 'postgres@mydb LOG duration ms statement INSERT INTO pgbench_history tid bid aid delta mtime VALUES CURRENT_TIMESTAMP', - fieldName: 'message', - fieldValue: - '2022-06-07 07:24:07.786 GMT [619164] postgres@mydb LOG: duration: 0.207 ms statement: INSERT INTO pgbench_history (tid, bid, aid, delta, mtime) VALUES (204, 41, 2624135, -2382, CURRENT_TIMESTAMP);', - doc_count: 21873, - bg_count: 0, - total_doc_count: 315172, - total_bg_count: 223766, - score: -13.815510557964274, - pValue: 0.000001, - normalizedScore: 0, - type: 'log_pattern', - }, - { - key: 'postgres@mydb LOG duration ms statement insert into bid values', - fieldName: 'message', - fieldValue: - '2022-06-07 07:23:12.509 GMT [619058] postgres@mydb LOG: duration: 0.057 ms statement: insert into pgbench_branches(bid,bbalance) values(10,0)', - doc_count: 22423, - bg_count: 0, - total_doc_count: 315172, - total_bg_count: 223766, - score: -13.815510557964274, - pValue: 0.000001, - normalizedScore: 0, - type: 'log_pattern', - }, - { - key: 'kubernetes.node.uid:641a2bef-4ea2-47c2-92f7-379ea1137e26', - type: 'keyword', - fieldName: 'kubernetes.node.uid', - fieldValue: '641a2bef-4ea2-47c2-92f7-379ea1137e26', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'host.mac:0e:56:50:16:32:5c', - type: 'keyword', - fieldName: 'host.mac', - fieldValue: '0e:56:50:16:32:5c', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'host.hostname:filebeat-t8pfz', - type: 'keyword', - fieldName: 'host.hostname', - fieldValue: 'filebeat-t8pfz', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.id:76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156', - type: 'keyword', - fieldName: 'container.id', - fieldValue: '76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'service.type:postgresql', - type: 'keyword', - fieldName: 'service.type', - fieldValue: 'postgresql', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'fileset.name:log', - type: 'keyword', - fieldName: 'fileset.name', - fieldValue: 'log', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'agent.hostname:filebeat-t8pfz', - type: 'keyword', - fieldName: 'agent.hostname', - fieldValue: 'filebeat-t8pfz', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'host.ip:10.40.5.185', - type: 'keyword', - fieldName: 'host.ip', - fieldValue: '10.40.5.185', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.node.hostname:gke-eden-3-staging-ssd-3-bc7684a4-2isy.c.elastic-product.internal', - type: 'keyword', - fieldName: 'kubernetes.node.hostname', - fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy.c.elastic-product.internal', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.pod.ip:10.40.5.6', - type: 'keyword', - fieldName: 'kubernetes.pod.ip', - fieldValue: '10.40.5.6', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'postgresql.log.database:pgbench', - type: 'keyword', - fieldName: 'postgresql.log.database', - fieldValue: 'pgbench', - doc_count: 150443, - bg_count: 0, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.container.name:postgres', - type: 'keyword', - fieldName: 'kubernetes.container.name', - fieldValue: 'postgres', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'log.file.path:/var/lib/docker/containers/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156-json.log', - type: 'keyword', - fieldName: 'log.file.path', - fieldValue: - '/var/lib/docker/containers/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156/76150c7488511e1b75e9ad4b9ebe2b7228198dc7d0e0cd0c65c5d6c7f953b156-json.log', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'cloud.instance.name:gke-eden-3-staging-ssd-3-bc7684a4-2isy', - type: 'keyword', - fieldName: 'cloud.instance.name', - fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.labels.io_kubernetes_container_name:postgres', - type: 'keyword', - fieldName: 'container.labels.io_kubernetes_container_name', - fieldValue: 'postgres', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.labels.pod-template-hash:5dc4ff9cf7', - type: 'keyword', - fieldName: 'kubernetes.labels.pod-template-hash', - fieldValue: '5dc4ff9cf7', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'log.level:LOG', - type: 'keyword', - fieldName: 'log.level', - fieldValue: 'LOG', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'postgresql.log.query:BEGIN;', - type: 'keyword', - fieldName: 'postgresql.log.query', - fieldValue: 'BEGIN;', - doc_count: 22214, - bg_count: 0, - total_doc_count: 178871, - total_bg_count: 221943, - score: 545.6888415128909, - pValue: 1.02411162216583e-237, - normalizedScore: 1, - }, - { - key: 'postgresql.log.query:END;', - type: 'keyword', - fieldName: 'postgresql.log.query', - fieldValue: 'END;', - doc_count: 20757, - bg_count: 0, - total_doc_count: 178871, - total_bg_count: 221943, - score: 489.1397423802259, - pValue: 3.709445585218203e-213, - normalizedScore: 1, - }, - { - key: 'event.module:postgresql', - type: 'keyword', - fieldName: 'event.module', - fieldValue: 'postgresql', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'service.name:postgres', - type: 'keyword', - fieldName: 'service.name', - fieldValue: 'postgres', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.pod.name:postgres-5dc4ff9cf7-flc4h', - type: 'keyword', - fieldName: 'kubernetes.pod.name', - fieldValue: 'postgres-5dc4ff9cf7-flc4h', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'event.timezone:GMT', - type: 'keyword', - fieldName: 'event.timezone', - fieldValue: 'GMT', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.node.labels.kubernetes_io/hostname:gke-eden-3-staging-ssd-3-bc7684a4-2isy', - type: 'keyword', - fieldName: 'kubernetes.node.labels.kubernetes_io/hostname', - fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.labels.app:postgres', - type: 'keyword', - fieldName: 'kubernetes.labels.app', - fieldValue: 'postgres', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'agent.name:filebeat-t8pfz', - type: 'keyword', - fieldName: 'agent.name', - fieldValue: 'filebeat-t8pfz', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'event.category:database', - type: 'keyword', - fieldName: 'event.category', - fieldValue: 'database', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.image.name:docker.elastic.co/sa/observe-postgres-hipster@sha256:d6e54ed8180612c98cfd7ea6fab35c5263c5bd7845d2d95d81af78e21febd6b4', - type: 'keyword', - fieldName: 'container.image.name', - fieldValue: - 'docker.elastic.co/sa/observe-postgres-hipster@sha256:d6e54ed8180612c98cfd7ea6fab35c5263c5bd7845d2d95d81af78e21febd6b4', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.labels.io_kubernetes_pod_uid:0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', - type: 'keyword', - fieldName: 'container.labels.io_kubernetes_pod_uid', - fieldValue: '0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'related.user:pgbench', - type: 'keyword', - fieldName: 'related.user', - fieldValue: 'pgbench', - doc_count: 150443, - bg_count: 0, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'agent.id:8a7d20ae-da85-4b50-9c7c-2b93ab794216', - type: 'keyword', - fieldName: 'agent.id', - fieldValue: '8a7d20ae-da85-4b50-9c7c-2b93ab794216', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.labels.io_kubernetes_sandbox_id:a7caeb81df949a4da817fe93879dfbe31e337fe059c128ea36e9661f88883427', - type: 'keyword', - fieldName: 'container.labels.io_kubernetes_sandbox_id', - fieldValue: 'a7caeb81df949a4da817fe93879dfbe31e337fe059c128ea36e9661f88883427', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.labels.annotation_io_kubernetes_container_hash:55d6093a', - type: 'keyword', - fieldName: 'container.labels.annotation_io_kubernetes_container_hash', - fieldValue: '55d6093a', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'user.name:pgbench', - type: 'keyword', - fieldName: 'user.name', - fieldValue: 'pgbench', - doc_count: 150443, - bg_count: 0, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'postgresql.log.query_step:statement', - type: 'keyword', - fieldName: 'postgresql.log.query_step', - fieldValue: 'statement', - doc_count: 155100, - bg_count: 40186, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'cloud.instance.id:2037777022342535795', - type: 'keyword', - fieldName: 'cloud.instance.id', - fieldValue: '2037777022342535795', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'stream:stderr', - type: 'keyword', - fieldName: 'stream', - fieldValue: 'stderr', - doc_count: 166214, - bg_count: 127186, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.replicaset.name:postgres-5dc4ff9cf7', - type: 'keyword', - fieldName: 'kubernetes.replicaset.name', - fieldValue: 'postgres-5dc4ff9cf7', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'agent.ephemeral_id:b66cd0b0-05cf-4d5a-bda9-7b27d5e80e2b', - type: 'keyword', - fieldName: 'agent.ephemeral_id', - fieldValue: 'b66cd0b0-05cf-4d5a-bda9-7b27d5e80e2b', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'event.kind:event', - type: 'keyword', - fieldName: 'event.kind', - fieldValue: 'event', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.name:k8s_postgres_postgres-5dc4ff9cf7-flc4h_observe-shop_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4_0', - type: 'keyword', - fieldName: 'container.name', - fieldValue: - 'k8s_postgres_postgres-5dc4ff9cf7-flc4h_observe-shop_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4_0', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'host.name:gke-eden-3-staging-ssd-3-bc7684a4-2isy', - type: 'keyword', - fieldName: 'host.name', - fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'ecs.version:1.12.0', - type: 'keyword', - fieldName: 'ecs.version', - fieldValue: '1.12.0', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.labels.io_kubernetes_pod_name:postgres-5dc4ff9cf7-flc4h', - type: 'keyword', - fieldName: 'container.labels.io_kubernetes_pod_name', - fieldValue: 'postgres-5dc4ff9cf7-flc4h', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.pod.uid:0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', - type: 'keyword', - fieldName: 'kubernetes.pod.uid', - fieldValue: '0ad8dd30-3fd2-4644-b46c-969c94ef7bc4', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'kubernetes.node.name:gke-eden-3-staging-ssd-3-bc7684a4-2isy', - type: 'keyword', - fieldName: 'kubernetes.node.name', - fieldValue: 'gke-eden-3-staging-ssd-3-bc7684a4-2isy', - doc_count: 165757, - bg_count: 123157, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'event.type:info', - type: 'keyword', - fieldName: 'event.type', - fieldValue: 'info', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'container.labels.io_kubernetes_container_logpath:/var/log/pods/observe-shop_postgres-5dc4ff9cf7-flc4h_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4/postgres/0.log', - type: 'keyword', - fieldName: 'container.labels.io_kubernetes_container_logpath', - fieldValue: - '/var/log/pods/observe-shop_postgres-5dc4ff9cf7-flc4h_0ad8dd30-3fd2-4644-b46c-969c94ef7bc4/postgres/0.log', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, - { - key: 'event.dataset:postgresql.log', - type: 'keyword', - fieldName: 'event.dataset', - fieldValue: 'postgresql.log', - doc_count: 157286, - bg_count: 59214, - total_doc_count: 178871, - total_bg_count: 221943, - score: 708.3964185322641, - pValue: 2.2250738585072626e-308, - normalizedScore: 1, - }, -]; diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/nav_control/index.tsx b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/nav_control/index.tsx index 4bb7f1404117c9..dbc42cf8459c13 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/nav_control/index.tsx +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/components/nav_control/index.tsx @@ -107,9 +107,9 @@ export function NavControl({}: {}) { }; }, [service.conversations]); - if (!isVisible) { - return null; - } + // if (!isVisible) { + // return null; + // } return ( <> diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts index 10195bf38651ef..1427c687da648f 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts @@ -8,6 +8,7 @@ import { useEffect, useState } from 'react'; import datemath from '@elastic/datemath'; import moment from 'moment'; +import { useLocalStorageListener } from '@kbn/ml-local-storage'; import { useKibana } from './use_kibana'; import { useObservabilityAIAssistantAppService } from './use_observability_ai_assistant_app_service'; @@ -22,6 +23,8 @@ export function useNavControlScreenContext() { }, } = useKibana(); + const [lsIndex] = useLocalStorageListener('obs-ai-assistant-index', null); + const [lsIndexTimeField] = useLocalStorageListener('obs-ai-assistant-index-time-field', null); const { from, to } = data.query.timefilter.timefilter.getTime(); const [href, setHref] = useState(window.location.href); @@ -62,8 +65,13 @@ export function useNavControlScreenContext() { const start = datemath.parse(from)?.toISOString() ?? moment().subtract(1, 'day').toISOString(); const end = datemath.parse(to)?.toISOString() ?? moment().toISOString(); + const index = + lsIndex !== null && lsIndexTimeField !== null + ? `The current Elasticsearch index is '${lsIndex}' and its time field is '${lsIndexTimeField}'.` + : `The current Elasticsearch index could not be identified or the current page does not make use of an Elasticsearch index.`; + return service.setScreenContext({ - screenDescription: `The user is looking at ${href}. The current time range is ${start} - ${end}.`, + screenDescription: `The user is looking at ${href}. The current time range is ${start} - ${end}. ${index}`, }); - }, [service, from, to, href]); + }, [service, from, to, href, lsIndex, lsIndexTimeField]); } From a2f0eb22de2592e3443db65e94e7fe6b393955b7 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 2 Apr 2024 12:49:04 +0200 Subject: [PATCH 06/31] add link to log rate analysis --- .../components/top_nav/discover_topnav.tsx | 5 +++++ .../get_aiops_log_rate_analysis_function.ts | 21 +++++++++++++++++-- .../hooks/use_nav_control_screen_context.ts | 5 +++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx index a1509338248372..a13ceaf9898e83 100644 --- a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx @@ -73,6 +73,10 @@ export const DiscoverTopNav = ({ const closeDataViewEditor = useRef<() => void | undefined>(); const [lsIndex, saveLsIndex] = useLocalStorageListener('obs-ai-assistant-index', null); + const [lsDataViewId, saveLsDataViewId] = useLocalStorageListener( + 'obs-ai-assistant-data-view-id', + null + ); const [lsIndexTimeField, saveLsIndexTimeField] = useLocalStorageListener( 'obs-ai-assistant-index-time-field', null @@ -192,6 +196,7 @@ export const DiscoverTopNav = ({ console.log('time field', dataView?.getTimeField()?.spec.name); saveLsIndex(dataView?.getIndexPattern() || null); saveLsIndexTimeField(dataView?.getTimeField()?.spec.name || null); + saveLsDataViewId(dataView?.id); }, [dataView]); const savedSearchId = useSavedSearch().id; diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index 4d2aea1c91e6d6..1535488a924e8a 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -9,6 +9,7 @@ import { queue } from 'async'; import { FromSchema } from 'json-schema-to-ts'; import { mean } from 'd3-array'; import moment from 'moment'; +import rison from '@kbn/rison'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -44,6 +45,10 @@ const parameters = { type: 'string', description: 'The Elasticsearch source index pattern.', }, + dataViewId: { + type: 'string', + description: 'The Kibana data view id.', + }, timefield: { type: 'string', description: 'The Elasticesarch source index pattern time field.', @@ -57,7 +62,7 @@ const parameters = { description: 'The end of the time range, in Elasticsearch date math, like `now-24h`.', }, }, - required: ['index', 'timefield', 'start', 'end'], + required: ['dataViewId', 'index', 'timefield', 'start', 'end'], } as const; export function registerGetAiopsLogRateAnalysisFunction({ @@ -75,7 +80,7 @@ export function registerGetAiopsLogRateAnalysisFunction({ defaultMessage: `Log rate analysis is a feature that uses advanced statistical methods to identify reasons for increases or decreases in log rates.`, } ), - description: `Log rate analysis is a Elastic AIOps feature that uses advanced statistical methods to identify reasons for increases or decreases in time series of log rates. The analysis returns significant field/value pairs found in the log rate change. The importance of field/value pairs is logIncrease descending. Briefly explain the data with value examples. Values with the same increase might correlate. Suggest actionable insights for remediations and identify potential security and performance issues.`, + description: `Log rate analysis is a Elastic AIOps feature that uses advanced statistical methods to identify reasons for increases or decreases in time series of log rates. The analysis returns significant field/value pairs found in the log rate change. The importance of field/value pairs is logIncrease descending. Briefly explain the data with value examples. Values with the same increase might correlate. Suggest actionable insights for remediations and identify potential security and performance issues. Finally, if available, provide a link to the log rate analysis UI.`, parameters, }, async ({ arguments: args }, abortSignal): Promise => { @@ -286,6 +291,18 @@ export function registerGetAiopsLogRateAnalysisFunction({ return { content: { + logRateAnalysisUILink: `/app/ml/aiops/log_rate_analysis?index=${ + args.dataViewId + }&_a=${rison.encode({ + logRateAnalysis: { + wp: { + bMin: wp.baselineMin, + bMax: wp.baselineMax, + dMin: wp.deviationMin, + dMax: wp.deviationMax, + }, + }, + })}`, logRateChange: { type: Object.keys(histogram.aggregations.change_point_request.type)[0], timestamp: histogram.aggregations.change_point_request.bucket.key, diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts index 1427c687da648f..c6e8f7a06ac6b0 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts @@ -24,6 +24,7 @@ export function useNavControlScreenContext() { } = useKibana(); const [lsIndex] = useLocalStorageListener('obs-ai-assistant-index', null); + const [lsDataViewId] = useLocalStorageListener('obs-ai-assistant-data-view-id', null); const [lsIndexTimeField] = useLocalStorageListener('obs-ai-assistant-index-time-field', null); const { from, to } = data.query.timefilter.timefilter.getTime(); @@ -71,7 +72,7 @@ export function useNavControlScreenContext() { : `The current Elasticsearch index could not be identified or the current page does not make use of an Elasticsearch index.`; return service.setScreenContext({ - screenDescription: `The user is looking at ${href}. The current time range is ${start} - ${end}. ${index}`, + screenDescription: `The user is looking at ${href}. The current time range is ${start} - ${end}. ${index} The Kibana Data View Id is ${lsDataViewId}.`, }); - }, [service, from, to, href, lsIndex, lsIndexTimeField]); + }, [service, from, to, href, lsDataViewId, lsIndex, lsIndexTimeField]); } From aa598e175033942c41efedb5a9a946fbc11224c4 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 2 Apr 2024 15:26:18 +0200 Subject: [PATCH 07/31] fix useEffect unmount --- .../main/components/top_nav/discover_topnav.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx index a13ceaf9898e83..e69759924a25c7 100644 --- a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx @@ -193,10 +193,16 @@ export const DiscoverTopNav = ({ ]); useEffect(() => { - console.log('time field', dataView?.getTimeField()?.spec.name); saveLsIndex(dataView?.getIndexPattern() || null); saveLsIndexTimeField(dataView?.getTimeField()?.spec.name || null); saveLsDataViewId(dataView?.id); + + return () => { + saveLsIndex(null); + saveLsIndexTimeField(null); + saveLsDataViewId(null); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps }, [dataView]); const savedSearchId = useSavedSearch().id; From e93fa4225e6b0ceee8e2d152b54a068fb6bfdb39 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Apr 2024 09:18:05 +0200 Subject: [PATCH 08/31] reenable log patterns --- .../assistant_functions/get_aiops_log_rate_analysis_function.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index 1535488a924e8a..5cb54aa2295482 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -235,7 +235,7 @@ export function registerGetAiopsLogRateAnalysisFunction({ const significantCategories: SignificantItem[] = []; // Get significant categories of text fields - if (textFieldCandidates.length > 100) { + if (textFieldCandidates.length > 0) { significantCategories.push( ...(await fetchSignificantCategories( esClient, From 7d7a55b1e051fe508c72a4324c5b8e99f4d0ec09 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Apr 2024 09:18:37 +0200 Subject: [PATCH 09/31] update dummy API endpoint --- .../aiops_api/server/routes/log_rate_analysis/define_route.ts | 2 +- .../server/routes/log_rate_analysis/route_handler_factory.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts index beced85b93a72b..d121ed70ac9f0a 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts @@ -29,7 +29,7 @@ export const defineRoute = ( coreStart: CoreStart, usageCounter?: UsageCounter ) => { - router.get( + router.post( { path: '/api/aiops/log_rate_analysis', validate: false, diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts index 650d1379721ed7..2b2d1fcf6d109d 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -57,7 +57,7 @@ export function routeHandlerFactory( // const executionContext = createExecutionContext(coreStart, AIOPS_PLUGIN_ID, request.route.path); return response.ok({ - body: [], + body: 'CAN HAS LOG RATE ANALYSIS', }); }; } From 69fe938455877200bf7f657db02db9ef0def86c4 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Apr 2024 09:19:33 +0200 Subject: [PATCH 10/31] LRA in Discover early days --- .../unified_histogram/public/chart/chart.tsx | 23 ++++++++++++++ .../public/chart/histogram.tsx | 30 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/plugins/unified_histogram/public/chart/chart.tsx b/src/plugins/unified_histogram/public/chart/chart.tsx index d8e8f73a40a8a0..ab883d04a4da3c 100644 --- a/src/plugins/unified_histogram/public/chart/chart.tsx +++ b/src/plugins/unified_histogram/public/chart/chart.tsx @@ -222,6 +222,29 @@ export function Chart({ const actions: IconButtonGroupProps['buttons'] = []; + if (!breakdown.field) { + actions.push({ + label: i18n.translate('unifiedHistogram.logRateAnalysisButton', { + defaultMessage: 'Run log rate analysis', + }), + iconType: 'stats', + isDisabled: isFlyoutVisible, + 'data-test-subj': 'unifiedHistogramLogRateAnalysis', + onClick: async () => { + console.log('services', services); + console.log('log rate analysis time range', getTimeRange()); + console.log('log rate analysis request', request); + console.log('log rate analysis chart', chart); + + const timeRange = getTimeRange(); + const resp = await services.http.post('/api/aiops/log_rate_analysis', { + body: {}, + }); + console.log('resp', resp); + }, + }); + } + if (canEditVisualizationOnTheFly) { actions.push({ label: i18n.translate('unifiedHistogram.editVisualizationButton', { diff --git a/src/plugins/unified_histogram/public/chart/histogram.tsx b/src/plugins/unified_histogram/public/chart/histogram.tsx index 50c8f7242c589a..a63a4facbc8459 100644 --- a/src/plugins/unified_histogram/public/chart/histogram.tsx +++ b/src/plugins/unified_histogram/public/chart/histogram.tsx @@ -181,6 +181,36 @@ export function Histogram({ visContext, onLoad, }); + console.log('lensProps', lensProps); + + const annotation = undefined; + if (annotation) { + lensProps.attributes.state.visualization.layers.push({ + layerId: '8d26ab67-b841-4877-9d02-55bf270f9caf', + layerType: 'annotations', + annotations: [ + { + type: 'manual', + icon: 'triangle', + textVisibility: true, + label: 'MY LABEL', + key: { + type: 'point_in_time', + timestamp: annotation.timestamp, + }, + id: 'a8fb297c-8d96-4011-93c0-45af110d5302', + isHidden: false, + color: '#F04E98', + lineStyle: 'solid', + lineWidth: 1, + outside: false, + }, + ], + // TODO check if we need to set filter from + // the filterManager + ignoreGlobalFilters: false, + }); + } const { euiTheme } = useEuiTheme(); const boxShadow = `0 2px 2px -1px ${euiTheme.colors.mediumShade}, From 32dd1846279dc707a267839df54f6eeb06576ade Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 8 May 2024 19:41:29 +0200 Subject: [PATCH 11/31] use setScreenContext instead of PoC localStorage approach --- .../components/layout/discover_layout.tsx | 19 +++++- .../components/top_nav/discover_topnav.tsx | 24 ------- x-pack/packages/ml/local_storage/index.ts | 1 - .../src/use_local_storage_listener.ts | 63 ------------------- .../hooks/use_nav_control_screen_context.ts | 13 +--- 5 files changed, 19 insertions(+), 101 deletions(-) delete mode 100644 x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts diff --git a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx index d4798089e09eea..c6ad8dd1811f40 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx @@ -142,12 +142,27 @@ export function DiscoverLayout({ stateContainer }: DiscoverLayoutProps) { }); // The assistant is getting the state from the url correctly - // expect from the index pattern where we have only the dataview id + // except from the index pattern where we have only the dataview id. useEffect(() => { + const indexPattern = dataView?.getIndexPattern() || null; + const indexPatternTimeField = dataView?.getTimeField()?.spec.name || null; + const dataViewId = dataView?.id; + + let indexPatternText = + 'The current Elasticsearch index pattern could not be identified or the current page does not make use of an Elasticsearch index pattern.'; + + if (indexPattern !== null && indexPatternTimeField === null) { + indexPatternText = `The current Elasticsearch index is '${indexPattern}' and it has no time field specified.`; + } else if (indexPattern !== null && indexPatternTimeField !== null) { + indexPatternText = `The current Elasticsearch index is '${indexPattern}' and its time field is '${indexPatternTimeField}'.`; + } + + const dataViewIdText = dataViewId ? ` The Kibana Data View Id is ${dataViewId}.` : ''; + return observabilityAIAssistant?.service.setScreenContext({ screenDescription: `The user is looking at the Discover view on the ${ isEsqlMode ? 'ES|QL' : 'dataView' - } mode. The index pattern is the ${dataView.getIndexPattern()}`, + } mode. ${indexPatternText}${dataViewIdText}`, }); }, [dataView, isEsqlMode, observabilityAIAssistant?.service]); diff --git a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx index e69759924a25c7..55fe8825477d9f 100644 --- a/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx +++ b/src/plugins/discover/public/application/main/components/top_nav/discover_topnav.tsx @@ -7,7 +7,6 @@ */ import React, { useCallback, useEffect, useMemo, useRef } from 'react'; -import { useLocalStorageListener } from '@kbn/ml-local-storage'; import { type DataView, DataViewType } from '@kbn/data-views-plugin/public'; import { DataViewPickerProps } from '@kbn/unified-search-plugin/public'; import { ENABLE_ESQL } from '@kbn/esql-utils'; @@ -72,16 +71,6 @@ export const DiscoverTopNav = ({ const closeFieldEditor = useRef<() => void | undefined>(); const closeDataViewEditor = useRef<() => void | undefined>(); - const [lsIndex, saveLsIndex] = useLocalStorageListener('obs-ai-assistant-index', null); - const [lsDataViewId, saveLsDataViewId] = useLocalStorageListener( - 'obs-ai-assistant-data-view-id', - null - ); - const [lsIndexTimeField, saveLsIndexTimeField] = useLocalStorageListener( - 'obs-ai-assistant-index-time-field', - null - ); - useEffect(() => { return () => { // Make sure to close the editors when unmounting @@ -192,19 +181,6 @@ export const DiscoverTopNav = ({ topNavMenu, ]); - useEffect(() => { - saveLsIndex(dataView?.getIndexPattern() || null); - saveLsIndexTimeField(dataView?.getTimeField()?.spec.name || null); - saveLsDataViewId(dataView?.id); - - return () => { - saveLsIndex(null); - saveLsIndexTimeField(null); - saveLsDataViewId(null); - }; - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [dataView]); - const savedSearchId = useSavedSearch().id; const savedSearchHasChanged = useSavedSearchHasChanged(); const dataViewPickerProps: DataViewPickerProps = useMemo(() => { diff --git a/x-pack/packages/ml/local_storage/index.ts b/x-pack/packages/ml/local_storage/index.ts index c4a41bebe328b4..f950f8791a3415 100644 --- a/x-pack/packages/ml/local_storage/index.ts +++ b/x-pack/packages/ml/local_storage/index.ts @@ -6,4 +6,3 @@ */ export { StorageContextProvider, useStorage } from './src/storage_context'; -export { useLocalStorageListener } from './src/use_local_storage_listener'; diff --git a/x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts b/x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts deleted file mode 100644 index 3bf82ba4edf728..00000000000000 --- a/x-pack/packages/ml/local_storage/src/use_local_storage_listener.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { useEffect, useState } from 'react'; - -const EVENT_NAME_PREFIX = 'local-storage-listener'; - -export function useLocalStorageListener(key: string, initValue: any) { - const [state, setState] = useState(() => { - const value = localStorage.getItem(key); - if (value !== null) { - try { - return JSON.parse(value); - } catch (e) { - return null; - } - } - - localStorage.setItem(key, JSON.stringify(initValue)); - window.dispatchEvent(new Event(`${EVENT_NAME_PREFIX}${key}`)); - return initValue; - }); - - useEffect(() => { - const value = localStorage.getItem(key); - if (value !== JSON.stringify(state)) { - localStorage.setItem(key, JSON.stringify(state)); - window.dispatchEvent(new Event(`${EVENT_NAME_PREFIX}${key}`)); - } - }, [key, state]); - - useEffect(() => { - const listenStorageChange = () => { - setState(() => { - const value = localStorage.getItem(key); - if (value !== null) { - try { - return JSON.parse(value); - } catch (e) { - return null; - } - } - - if (initValue !== null) { - localStorage.setItem(key, JSON.stringify(initValue)); - window.dispatchEvent(new Event(`${EVENT_NAME_PREFIX}${key}`)); - return initValue; - } - - return null; - }); - }; - window.addEventListener(`${EVENT_NAME_PREFIX}${key}`, listenStorageChange); - return () => window.removeEventListener(`${EVENT_NAME_PREFIX}${key}`, listenStorageChange); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []); - - return [state, setState]; -} diff --git a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts index c6e8f7a06ac6b0..10195bf38651ef 100644 --- a/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts +++ b/x-pack/plugins/observability_solution/observability_ai_assistant_app/public/hooks/use_nav_control_screen_context.ts @@ -8,7 +8,6 @@ import { useEffect, useState } from 'react'; import datemath from '@elastic/datemath'; import moment from 'moment'; -import { useLocalStorageListener } from '@kbn/ml-local-storage'; import { useKibana } from './use_kibana'; import { useObservabilityAIAssistantAppService } from './use_observability_ai_assistant_app_service'; @@ -23,9 +22,6 @@ export function useNavControlScreenContext() { }, } = useKibana(); - const [lsIndex] = useLocalStorageListener('obs-ai-assistant-index', null); - const [lsDataViewId] = useLocalStorageListener('obs-ai-assistant-data-view-id', null); - const [lsIndexTimeField] = useLocalStorageListener('obs-ai-assistant-index-time-field', null); const { from, to } = data.query.timefilter.timefilter.getTime(); const [href, setHref] = useState(window.location.href); @@ -66,13 +62,8 @@ export function useNavControlScreenContext() { const start = datemath.parse(from)?.toISOString() ?? moment().subtract(1, 'day').toISOString(); const end = datemath.parse(to)?.toISOString() ?? moment().toISOString(); - const index = - lsIndex !== null && lsIndexTimeField !== null - ? `The current Elasticsearch index is '${lsIndex}' and its time field is '${lsIndexTimeField}'.` - : `The current Elasticsearch index could not be identified or the current page does not make use of an Elasticsearch index.`; - return service.setScreenContext({ - screenDescription: `The user is looking at ${href}. The current time range is ${start} - ${end}. ${index} The Kibana Data View Id is ${lsDataViewId}.`, + screenDescription: `The user is looking at ${href}. The current time range is ${start} - ${end}.`, }); - }, [service, from, to, href, lsDataViewId, lsIndex, lsIndexTimeField]); + }, [service, from, to, href]); } From 2815f3a6be88e8e1312a566cc5c92ba8d2eee65c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 13 May 2024 10:38:25 +0200 Subject: [PATCH 12/31] discover start prompt --- .../main/components/layout/discover_layout.tsx | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx index c6ad8dd1811f40..99b28538ef3341 100644 --- a/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx +++ b/src/plugins/discover/public/application/main/components/layout/discover_layout.tsx @@ -163,6 +163,17 @@ export function DiscoverLayout({ stateContainer }: DiscoverLayoutProps) { screenDescription: `The user is looking at the Discover view on the ${ isEsqlMode ? 'ES|QL' : 'dataView' } mode. ${indexPatternText}${dataViewIdText}`, + starterPrompts: [ + { + title: i18n.translate('discover.aiAssistant.starterPrompts.logRateAnalysis.title', { + defaultMessage: 'Explain', + }), + prompt: i18n.translate('discover.aiAssistant.starterPrompts.logRateAnalysis.title', { + defaultMessage: 'Can you analyze the log rate?', + }), + icon: 'inspect', + }, + ], }); }, [dataView, isEsqlMode, observabilityAIAssistant?.service]); From 9c5cf2ddb98f6ef445fbb8f1a17b63240b2aaa8b Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 13 May 2024 10:40:18 +0200 Subject: [PATCH 13/31] aiops ai assistant ui --- x-pack/plugins/aiops_api/common/types.ts | 46 ++++ x-pack/plugins/aiops_api/jest.config.js | 2 +- x-pack/plugins/aiops_api/kibana.jsonc | 6 +- .../aiops_api/public/functions/index.ts | 20 ++ .../public/functions/log_rate_analysis.tsx | 213 ++++++++++++++++++ x-pack/plugins/aiops_api/public/index.ts | 12 + x-pack/plugins/aiops_api/public/plugin.ts | 51 +++++ x-pack/plugins/aiops_api/public/types.ts | 46 ++++ .../get_aiops_log_rate_analysis_function.ts | 71 ++++-- 9 files changed, 441 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugins/aiops_api/common/types.ts create mode 100644 x-pack/plugins/aiops_api/public/functions/index.ts create mode 100644 x-pack/plugins/aiops_api/public/functions/log_rate_analysis.tsx create mode 100644 x-pack/plugins/aiops_api/public/index.ts create mode 100644 x-pack/plugins/aiops_api/public/plugin.ts create mode 100644 x-pack/plugins/aiops_api/public/types.ts diff --git a/x-pack/plugins/aiops_api/common/types.ts b/x-pack/plugins/aiops_api/common/types.ts new file mode 100644 index 00000000000000..dd1cc5f9ef82b6 --- /dev/null +++ b/x-pack/plugins/aiops_api/common/types.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { SignificantItem } from '@kbn/ml-agg-utils'; + +export interface GetAiopsLogRateAnalysisFunctionResponse { + content: + | string + | { + logRateChange: { + type: string; + spikeLogRate?: number; + dipLogRate?: number; + averageLogRate: number; + logRateAggregationIntervalUsedForAnalysis: string; + documentSamplingFactorForAnalysis?: number; + }; + significantItems: Array<{ + field: string; + value: string | number; + type: 'metadata' | 'log message pattern'; + documentCount: number; + baselineCount: number; + logIncrease: string; + }>; + }; + data: { + dateHistogram: Record; + logRateAnalysisUILink: string; + logRateChange: { + type: string; + timestamp: string; + spikeLogRate?: number; + dipLogRate?: number; + averageLogRate: number; + logRateAggregationIntervalUsedForAnalysis: string; + documentSamplingFactorForAnalysis?: number; + extendedChangePoint: { startTs: number; endTs: number }; + }; + significantItems: SignificantItem[]; + }; +} diff --git a/x-pack/plugins/aiops_api/jest.config.js b/x-pack/plugins/aiops_api/jest.config.js index fec51ff18afbc8..496ef3d02f9e7c 100644 --- a/x-pack/plugins/aiops_api/jest.config.js +++ b/x-pack/plugins/aiops_api/jest.config.js @@ -11,5 +11,5 @@ module.exports = { roots: ['/x-pack/plugins/aiops_api'], coverageDirectory: '/target/kibana-coverage/jest/x-pack/plugins/aiops_api', coverageReporters: ['text', 'html'], - collectCoverageFrom: ['/x-pack/plugins/aiops_api/server/**/*.{js,ts,tsx}'], + collectCoverageFrom: ['/x-pack/plugins/aiops_api/{public,server}/**/*.{js,ts,tsx}'], }; diff --git a/x-pack/plugins/aiops_api/kibana.jsonc b/x-pack/plugins/aiops_api/kibana.jsonc index bcebc78072cbc0..d07b08cbf02e7d 100644 --- a/x-pack/plugins/aiops_api/kibana.jsonc +++ b/x-pack/plugins/aiops_api/kibana.jsonc @@ -6,12 +6,14 @@ "plugin": { "id": "aiopsApi", "server": true, - "browser": false, + "browser": true, "requiredPlugins": [ "licensing", "observabilityAIAssistant" ], "optionalPlugins": [], - "requiredBundles": [] + "requiredBundles": [ + "charts" + ] } } diff --git a/x-pack/plugins/aiops_api/public/functions/index.ts b/x-pack/plugins/aiops_api/public/functions/index.ts new file mode 100644 index 00000000000000..8ecc81491828c4 --- /dev/null +++ b/x-pack/plugins/aiops_api/public/functions/index.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { RegisterRenderFunctionDefinition } from '@kbn/observability-ai-assistant-plugin/public/types'; +import type { AiopsApiPluginStartDeps } from '../types'; +import { registerLogRateAnalysisRenderFunction } from './log_rate_analysis'; + +export async function registerFunctions({ + registerRenderFunction, + pluginsStart, +}: { + registerRenderFunction: RegisterRenderFunctionDefinition; + pluginsStart: AiopsApiPluginStartDeps; +}) { + registerLogRateAnalysisRenderFunction({ pluginsStart, registerRenderFunction }); +} diff --git a/x-pack/plugins/aiops_api/public/functions/log_rate_analysis.tsx b/x-pack/plugins/aiops_api/public/functions/log_rate_analysis.tsx new file mode 100644 index 00000000000000..578bbb194b9d99 --- /dev/null +++ b/x-pack/plugins/aiops_api/public/functions/log_rate_analysis.tsx @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import { Axis, Chart, HistogramBarSeries, Position, ScaleType, Settings } from '@elastic/charts'; +import type { BarStyleAccessor } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; +import type { EuiBasicTableColumn } from '@elastic/eui'; +import { EuiInMemoryTable, EuiCode, EuiSpacer, EuiText } from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import type { + RegisterRenderFunctionDefinition, + RenderFunction, +} from '@kbn/observability-ai-assistant-plugin/public'; +import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common'; + +import type { AiopsApiPluginStartDeps } from '../types'; + +import type { GetAiopsLogRateAnalysisFunctionResponse } from '../../common/types'; + +const LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR = 'orange'; +const SPEC_ID = 'document_count'; +const NARROW_COLUMN_WIDTH = '120px'; + +const overallSeriesName = i18n.translate( + 'xpack.aiops.dataGrid.field.documentCountChart.seriesLabel', + { + defaultMessage: 'document count', + } +); + +export function registerLogRateAnalysisRenderFunction({ + registerRenderFunction, + pluginsStart, +}: { + registerRenderFunction: RegisterRenderFunctionDefinition; + pluginsStart: AiopsApiPluginStartDeps; +}) { + console.log('REGISTER FUNCTION'); + const renderFunction: RenderFunction<{}, GetAiopsLogRateAnalysisFunctionResponse> = ({ + arguments: args, + response, + }) => { + console.log('response', response); + + if (typeof response.content === 'string') { + return null; + } + + // CHART + + const adjustedChartPoints = Object.entries(response.data.dateHistogram).map( + ([time, value]) => ({ + time: Number(time), + value, + }) + ); + + // const xAxisFormatter = fieldFormats.deserialize({ id: 'date' }); + const useLegacyTimeAxis = false; + + const barStyle = { + rect: { + opacity: 1, + fill: LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, + }, + }; + + const extendedChangePoint = response.data.logRateChange.extendedChangePoint; + + // Used to highlight an auto-detected change point in the date histogram. + const barStyleAccessor: BarStyleAccessor = (d, g) => { + return g.specId === 'document_count' && + extendedChangePoint && + d.x > extendedChangePoint.startTs && + d.x < extendedChangePoint.endTs + ? barStyle + : null; + }; + + // TABLE + + const tableItems = response.content.significantItems; + + const columns: Array> = [ + { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldName', + field: 'field', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldNameLabel', { + defaultMessage: 'Field name', + }), + render: (_, { field }) => { + return ( + + {field} + + ); + }, + sortable: true, + valign: 'middle', + }, + { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldValue', + field: 'value', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldValueLabel', { + defaultMessage: 'Field value', + }), + render: (_, { value, type }) => ( + + {type === 'metadata' ? ( + String(value) + ) : ( + + + {String(value)} + + + )} + + ), + sortable: true, + textOnly: true, + truncateText: { lines: 3 }, + valign: 'middle', + }, + { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnDocCount', + width: NARROW_COLUMN_WIDTH, + field: 'logIncrease', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.docCountLabel', { + defaultMessage: 'Increase', + }), + sortable: true, + valign: 'middle', + }, + ]; + + const sorting = { + sort: { + field: 'logIncrease', + direction: 'desc' as const, + }, + }; + + return ( +
+ + + + xAxisFormatter.convert(value)} + labelFormat={useLegacyTimeAxis ? undefined : () => ''} + timeAxisLayerCount={useLegacyTimeAxis ? 0 : 2} + style={useLegacyTimeAxis ? {} : MULTILAYER_TIME_AXIS_STYLE} + /> + {adjustedChartPoints?.length && ( + + )} + + + + +

+ + More analysis options available in{' '} + Log Rate Analysis. + +

+
+ ); + }; + registerRenderFunction('get_aiops_log_rate_analysis', renderFunction); +} diff --git a/x-pack/plugins/aiops_api/public/index.ts b/x-pack/plugins/aiops_api/public/index.ts new file mode 100644 index 00000000000000..e73c2fc23f55da --- /dev/null +++ b/x-pack/plugins/aiops_api/public/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { AiopsApiPlugin } from './plugin'; + +export function plugin() { + return new AiopsApiPlugin(); +} diff --git a/x-pack/plugins/aiops_api/public/plugin.ts b/x-pack/plugins/aiops_api/public/plugin.ts new file mode 100644 index 00000000000000..39ac7e3c787c6d --- /dev/null +++ b/x-pack/plugins/aiops_api/public/plugin.ts @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { CoreStart, Plugin } from '@kbn/core/public'; +import { type CoreSetup } from '@kbn/core/public'; +import { firstValueFrom } from 'rxjs'; +import type { + AiopsApiPluginSetup, + AiopsApiPluginSetupDeps, + AiopsApiPluginStart, + AiopsApiPluginStartDeps, +} from './types'; + +export type AiopsCoreSetup = CoreSetup; + +export class AiopsApiPlugin + implements + Plugin< + AiopsApiPluginSetup, + AiopsApiPluginStart, + AiopsApiPluginSetupDeps, + AiopsApiPluginStartDeps + > +{ + public setup(core: AiopsCoreSetup, { licensing }: AiopsApiPluginSetupDeps) { + Promise.all([firstValueFrom(licensing.license$), core.getStartServices()]).then( + ([license, [coreStart, pluginStart]]) => { + if (license.hasAtLeast('platinum')) { + // Do something here + } + } + ); + } + + public start(core: CoreStart, pluginsStart: AiopsApiPluginStartDeps): AiopsApiPluginStart { + console.log('======== REGISTER!!!!'); + const service = pluginsStart.observabilityAIAssistant.service; + + service.register(async ({ registerRenderFunction }) => { + const { registerFunctions } = await import('./functions'); + + await registerFunctions({ pluginsStart, registerRenderFunction }); + }); + } + + public stop() {} +} diff --git a/x-pack/plugins/aiops_api/public/types.ts b/x-pack/plugins/aiops_api/public/types.ts new file mode 100644 index 00000000000000..83a956349f42d0 --- /dev/null +++ b/x-pack/plugins/aiops_api/public/types.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ChartsPluginStart } from '@kbn/charts-plugin/public'; +import type { ExecutionContextStart } from '@kbn/core-execution-context-browser'; +import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; +import type { SecurityPluginStart, SecurityPluginSetup } from '@kbn/security-plugin/public'; +import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; +import type { LensPublicStart } from '@kbn/lens-plugin/public'; +import type { LicensingPluginSetup, LicensingPluginStart } from '@kbn/licensing-plugin/public'; +import type { SharePluginStart } from '@kbn/share-plugin/public'; +import type { UiActionsStart } from '@kbn/ui-actions-plugin/public'; +import type { + ObservabilityAIAssistantPublicSetup, + ObservabilityAIAssistantPublicStart, +} from '@kbn/observability-ai-assistant-plugin/public'; +import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; +import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public'; + +export interface AiopsApiPluginSetupDeps { + licensing: LicensingPluginSetup; + security: SecurityPluginSetup; + observabilityAIAssistant: ObservabilityAIAssistantPublicSetup; +} + +export interface AiopsApiPluginStartDeps { + licensing: LicensingPluginStart; + share: SharePluginStart; + security: SecurityPluginStart; + lens: LensPublicStart; + data: DataPublicPluginStart; + charts: ChartsPluginStart; + uiActions: UiActionsStart; + unifiedSearch: UnifiedSearchPublicPluginStart; + storage: IStorageWrapper; + executionContext: ExecutionContextStart; + usageCollection: UsageCollectionSetup; + observabilityAIAssistant: ObservabilityAIAssistantPublicStart; +} + +export type AiopsApiPluginSetup = void; +export type AiopsApiPluginStart = void; diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index 5cb54aa2295482..dcecbe979baccc 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -9,10 +9,11 @@ import { queue } from 'async'; import { FromSchema } from 'json-schema-to-ts'; import { mean } from 'd3-array'; import moment from 'moment'; -import rison from '@kbn/rison'; +import dedent from 'dedent'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import rison from '@kbn/rison'; import { type SignificantItem } from '@kbn/ml-agg-utils'; import { i18n } from '@kbn/i18n'; import dateMath from '@kbn/datemath'; @@ -26,6 +27,8 @@ import { fetchSignificantCategories } from '@kbn/aiops-log-rate-analysis/queries import { fetchSignificantTermPValues } from '@kbn/aiops-log-rate-analysis/queries/fetch_significant_term_p_values'; import { getSampleProbability } from '@kbn/ml-random-sampler-utils'; +import type { GetAiopsLogRateAnalysisFunctionResponse } from '../../common/types'; + import { FunctionRegistrationParameters } from '.'; // import { ApmTimeseries, getApmTimeseries } from '../routes/assistant_functions/get_apm_timeseries'; @@ -72,7 +75,7 @@ export function registerGetAiopsLogRateAnalysisFunction({ }: FunctionRegistrationParameters) { registerFunction( { - contexts: ['aiops'], + // contexts: ['aiops'], name: 'get_aiops_log_rate_analysis', descriptionForUser: i18n.translate( 'xpack.aiops.observabilityAiAssistant.functions.registerGetAiopsLogRateAnalyssi.descriptionForUser', @@ -80,7 +83,18 @@ export function registerGetAiopsLogRateAnalysisFunction({ defaultMessage: `Log rate analysis is a feature that uses advanced statistical methods to identify reasons for increases or decreases in log rates.`, } ), - description: `Log rate analysis is a Elastic AIOps feature that uses advanced statistical methods to identify reasons for increases or decreases in time series of log rates. The analysis returns significant field/value pairs found in the log rate change. The importance of field/value pairs is logIncrease descending. Briefly explain the data with value examples. Values with the same increase might correlate. Suggest actionable insights for remediations and identify potential security and performance issues. Finally, if available, provide a link to the log rate analysis UI.`, + description: dedent(` + Log rate analysis is an Elastic AIOps feature to identify causes of spike/dips in time series of log rates. The analysis returns significant field/value pairs found in the log rate change sorted by logIncrease descending. + + Users will see a UI with a chart of the log rate over time, with the log rate change highlighted. The UI will also include a table with all identified significant field/value pairs. + + Your task is to summarize the provided data and provide context about the given data on display in the UI: + + - Infer and explain the environment the data was obtained from, summarize the most contributing field/value pairs and provide some field/value pair examples. + - If the data hints at a security or performance issue, explain the root cause and 1-2 steps to remediate the problem. + + Your output should be very concise, non-repeating, to-the-point. Your audience are technical users like SREs or Security Analysts using Elasticsearch and Kibana. + `), parameters, }, async ({ arguments: args }, abortSignal): Promise => { @@ -90,12 +104,12 @@ export function registerGetAiopsLogRateAnalysisFunction({ // CHANGE POINT DETECTION - const barTarget = 70; + const barTarget = 75; const earliestMs = dateMath.parse(args.start)?.valueOf(); const latestMs = dateMath.parse(args.end, { roundUp: true })?.valueOf(); if (earliestMs === undefined || latestMs === undefined) { - return { content: 'Could not parse time range.', data: [] }; + return { content: 'Could not parse time range.', data: { significantItems: [] } }; } const delta = latestMs - earliestMs; const dayMs = 86400 * 1000; @@ -289,23 +303,22 @@ export function registerGetAiopsLogRateAnalysisFunction({ const debugDelta = (debugEndTime - debugStartTime) / 1000; console.log(`Took: ${debugDelta}s`); + const logRateChange = { + type: Object.keys(histogram.aggregations.change_point_request.type)[0], + timestamp: histogram.aggregations.change_point_request.bucket.key, + [logRateAttributeName]: histogram.aggregations.change_point_request.bucket.doc_count, + averageLogRate: Math.round(mean(Object.values(buckets)) ?? 0), + logRateAggregationIntervalUsedForAnalysis: moment + .duration(Math.round(intervalMs / 1000), 'seconds') + .humanize(), + ...(sampleProbability < 1 ? { documentSamplingFactorForAnalysis: sampleProbability } : {}), + extendedChangePoint, + }; + return { content: { - logRateAnalysisUILink: `/app/ml/aiops/log_rate_analysis?index=${ - args.dataViewId - }&_a=${rison.encode({ - logRateAnalysis: { - wp: { - bMin: wp.baselineMin, - bMax: wp.baselineMax, - dMin: wp.deviationMin, - dMax: wp.deviationMax, - }, - }, - })}`, logRateChange: { type: Object.keys(histogram.aggregations.change_point_request.type)[0], - timestamp: histogram.aggregations.change_point_request.bucket.key, [logRateAttributeName]: histogram.aggregations.change_point_request.bucket.doc_count, averageLogRate: Math.round(mean(Object.values(buckets)) ?? 0), logRateAggregationIntervalUsedForAnalysis: moment @@ -331,14 +344,26 @@ export function registerGetAiopsLogRateAnalysisFunction({ : `${doc_count} documents up from 0 documents in baseline`, })), }, - data: [...significantTerms, ...significantCategories], + data: { + dateHistogram: buckets, + logRateAnalysisUILink: `/app/ml/aiops/log_rate_analysis?index=${ + args.dataViewId + }&_a=${rison.encode({ + logRateAnalysis: { + wp: { + bMin: wp.baselineMin, + bMax: wp.baselineMax, + dMin: wp.deviationMin, + dMax: wp.deviationMax, + }, + }, + })}`, + logRateChange, + significantItems: [...significantTerms, ...significantCategories], + }, }; } ); } export type GetAiopsLogRateAnalysisFunctionArguments = FromSchema; -export interface GetAiopsLogRateAnalysisFunctionResponse { - content: any; - data: typeof significantTerms; -} From 9134f58ec9642e015fac846bdc5b30c53bef4c7e Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 May 2024 10:55:16 +0200 Subject: [PATCH 14/31] aiops_components: refactor document count chart --- x-pack/packages/ml/aiops_components/index.ts | 10 ++++-- .../src/document_count_chart/bar_style.ts | 31 +++++++++++++++++ .../src/document_count_chart/constants.ts | 8 +++++ .../document_count_chart.tsx | 33 ++++++++----------- .../src/document_count_chart/i18n.ts | 22 +++++++++++++ .../src/document_count_chart/index.ts | 9 ++++- .../log_rate_analysis_content.tsx | 18 ++-------- 7 files changed, 92 insertions(+), 39 deletions(-) create mode 100644 x-pack/packages/ml/aiops_components/src/document_count_chart/bar_style.ts create mode 100644 x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts create mode 100644 x-pack/packages/ml/aiops_components/src/document_count_chart/i18n.ts diff --git a/x-pack/packages/ml/aiops_components/index.ts b/x-pack/packages/ml/aiops_components/index.ts index c5798bc4b0ae07..181fc1f15d3c82 100644 --- a/x-pack/packages/ml/aiops_components/index.ts +++ b/x-pack/packages/ml/aiops_components/index.ts @@ -8,8 +8,14 @@ export { DualBrush, DualBrushAnnotation } from './src/dual_brush'; export { ProgressControls } from './src/progress_controls'; export { + documentCountChartOverallSeriesName, DocumentCountChart, DocumentCountChartRedux, - type BrushSettings, + DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID, + getLogRateAnalysisBarStyleAccessor, +} from './src/document_count_chart'; +export type { + BrushSettings, + BrushSelectionUpdateHandler, + DocumentCountChartProps, } from './src/document_count_chart'; -export type { DocumentCountChartProps } from './src/document_count_chart'; diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/bar_style.ts b/x-pack/packages/ml/aiops_components/src/document_count_chart/bar_style.ts new file mode 100644 index 00000000000000..f343f984ee2a63 --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/bar_style.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { BarStyleAccessor } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; + +import { LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR } from '@kbn/aiops-log-rate-analysis'; +import type { DocumentCountStatsChangePoint } from '@kbn/aiops-log-rate-analysis'; + +import { DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID } from './constants'; + +const logRateAnalysisHighlightedBarStyle = { + rect: { + opacity: 1, + fill: LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, + }, +}; + +export const getLogRateAnalysisBarStyleAccessor = + (changePoint: DocumentCountStatsChangePoint): BarStyleAccessor => + (d, g) => { + return g.specId === DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID && + changePoint && + d.x > changePoint.startTs && + d.x < changePoint.endTs + ? logRateAnalysisHighlightedBarStyle + : null; + }; diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts b/x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts new file mode 100644 index 00000000000000..2260d0d6282bfc --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID = 'document_count'; diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx index fbfece79b3d77c..a16531f842005a 100644 --- a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx @@ -41,6 +41,11 @@ import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { DualBrush, DualBrushAnnotation } from '../..'; import { BrushBadge } from './brush_badge'; +import { DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID } from './constants'; +import { + documentCountChartOverallSeriesName, + documentCountChartOverallSeriesNameWithSplit, +} from './i18n'; declare global { interface Window { @@ -82,7 +87,7 @@ type SetAutoRunAnalysisFn = (isAutoRun: boolean) => void; /** * Brush selection update handler */ -type BrushSelectionUpdateHandler = ( +export type BrushSelectionUpdateHandler = ( /** Payload for the brush selection update */ d: BrushSelectionUpdatePayload ) => void; @@ -138,8 +143,6 @@ export interface DocumentCountChartProps { changePoint?: DocumentCountStatsChangePoint; } -const SPEC_ID = 'document_count'; - const BADGE_HEIGHT = 20; const BADGE_WIDTH = 75; @@ -199,20 +202,6 @@ export const DocumentCountChart: FC = (props) => { const xAxisFormatter = fieldFormats.deserialize({ id: 'date' }); const useLegacyTimeAxis = uiSettings.get('visualization:useLegacyTimeAxis', false); - const overallSeriesName = i18n.translate( - 'xpack.aiops.dataGrid.field.documentCountChart.seriesLabel', - { - defaultMessage: 'document count', - } - ); - - const overallSeriesNameWithSplit = i18n.translate( - 'xpack.aiops.dataGrid.field.documentCountChartSplit.seriesLabel', - { - defaultMessage: 'Other document count', - } - ); - // TODO Let user choose between ZOOM and BRUSH mode. const [viewMode] = useState(VIEW_MODE.BRUSH); @@ -503,8 +492,12 @@ export const DocumentCountChart: FC = (props) => { /> {adjustedChartPoints?.length && ( = (props) => { )} {adjustedChartPointsSplit?.length && ( = ({ dispatch(clearAllRowState()); } - const barStyle = { - rect: { - opacity: 1, - fill: LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, - }, - }; - // Used to highlight an auto-detected change point in the date histogram. const barStyleAccessor: BarStyleAccessor | undefined = isBrushCleared && documentCountStats?.changePoint - ? (d, g) => { - return g.specId === 'document_count' && - documentCountStats?.changePoint && - d.x > documentCountStats.changePoint.startTs && - d.x < documentCountStats.changePoint.endTs - ? barStyle - : null; - } + ? getLogRateAnalysisBarStyleAccessor(documentCountStats.changePoint) : undefined; const triggerAnalysisForManualSelection = useCallback(() => { From 80a179b1b64ae6a53b4bf50311f0aace79bcda39 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 May 2024 12:28:18 +0200 Subject: [PATCH 15/31] aiops_components: simple document count chart --- x-pack/packages/ml/aiops_components/index.ts | 1 + .../src/document_count_chart/index.ts | 1 + .../simple_document_count_chart.tsx | 95 +++++++++++++ .../src/results_table/constants.ts | 8 ++ .../src/results_table/index.ts | 8 ++ x-pack/plugins/aiops_api/common/types.ts | 3 - x-pack/plugins/aiops_api/kibana.jsonc | 5 +- ...is.tsx => get_aiops_log_rate_analysis.tsx} | 127 ++++-------------- .../aiops_api/public/functions/index.ts | 7 +- x-pack/plugins/aiops_api/public/plugin.ts | 26 ++-- x-pack/plugins/aiops_api/public/types.ts | 12 +- .../get_aiops_log_rate_analysis_function.ts | 9 +- x-pack/plugins/aiops_api/server/types.ts | 4 + 13 files changed, 169 insertions(+), 137 deletions(-) create mode 100644 x-pack/packages/ml/aiops_components/src/document_count_chart/simple_document_count_chart.tsx create mode 100644 x-pack/packages/ml/aiops_components/src/results_table/constants.ts create mode 100644 x-pack/packages/ml/aiops_components/src/results_table/index.ts rename x-pack/plugins/aiops_api/public/functions/{log_rate_analysis.tsx => get_aiops_log_rate_analysis.tsx} (52%) diff --git a/x-pack/packages/ml/aiops_components/index.ts b/x-pack/packages/ml/aiops_components/index.ts index 181fc1f15d3c82..1b032e17a755d4 100644 --- a/x-pack/packages/ml/aiops_components/index.ts +++ b/x-pack/packages/ml/aiops_components/index.ts @@ -11,6 +11,7 @@ export { documentCountChartOverallSeriesName, DocumentCountChart, DocumentCountChartRedux, + SimpleDocumentCountChart, DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID, getLogRateAnalysisBarStyleAccessor, } from './src/document_count_chart'; diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts b/x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts index 49c2733fd1e9a7..e7070b73d776da 100644 --- a/x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/index.ts @@ -15,3 +15,4 @@ export type { DocumentCountChartProps, } from './document_count_chart'; export { documentCountChartOverallSeriesName } from './i18n'; +export { SimpleDocumentCountChart } from './simple_document_count_chart'; diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/simple_document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/simple_document_count_chart.tsx new file mode 100644 index 00000000000000..6f378b8a6f04fd --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/simple_document_count_chart.tsx @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FC } from 'react'; + +import { Axis, Chart, HistogramBarSeries, Position, ScaleType, Settings } from '@elastic/charts'; + +import { i18n } from '@kbn/i18n'; +import type { IUiSettingsClient } from '@kbn/core/public'; +import type { ChartsPluginStart } from '@kbn/charts-plugin/public'; +import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; +import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common'; +import type { DocumentCountStatsChangePoint } from '@kbn/aiops-log-rate-analysis'; + +import { documentCountChartOverallSeriesName } from './i18n'; +import { getLogRateAnalysisBarStyleAccessor } from './bar_style'; +import { DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID } from './constants'; + +interface SimpleDocumentCountChartProps { + /** List of Kibana services that are required as dependencies */ + dependencies: { + charts: ChartsPluginStart; + fieldFormats: FieldFormatsStart; + uiSettings: IUiSettingsClient; + }; + dateHistogram: Record; + changePoint: DocumentCountStatsChangePoint; +} + +export const SimpleDocumentCountChart: FC = ({ + dependencies, + dateHistogram, + changePoint, +}) => { + const { uiSettings, fieldFormats, charts } = dependencies; + + const chartBaseTheme = charts.theme.useChartsBaseTheme(); + + const xAxisFormatter = fieldFormats.deserialize({ id: 'date' }); + const useLegacyTimeAxis = uiSettings.get('visualization:useLegacyTimeAxis', false); + + // Used to highlight an auto-detected change point in the date histogram. + const barStyleAccessor = getLogRateAnalysisBarStyleAccessor(changePoint); + + const adjustedChartPoints = Object.entries(dateHistogram).map(([time, value]) => ({ + time: Number(time), + value, + })); + + return ( + + + + xAxisFormatter.convert(value)} + labelFormat={useLegacyTimeAxis ? undefined : () => ''} + timeAxisLayerCount={useLegacyTimeAxis ? 0 : 2} + style={useLegacyTimeAxis ? {} : MULTILAYER_TIME_AXIS_STYLE} + /> + {adjustedChartPoints?.length && ( + + )} + + ); +}; diff --git a/x-pack/packages/ml/aiops_components/src/results_table/constants.ts b/x-pack/packages/ml/aiops_components/src/results_table/constants.ts new file mode 100644 index 00000000000000..59b9b2fc1e0674 --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/results_table/constants.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export const NARROW_COLUMN_WIDTH = '120px'; diff --git a/x-pack/packages/ml/aiops_components/src/results_table/index.ts b/x-pack/packages/ml/aiops_components/src/results_table/index.ts new file mode 100644 index 00000000000000..e4f592441a35ee --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/results_table/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { NARROW_COLUMN_WIDTH } from './constants'; diff --git a/x-pack/plugins/aiops_api/common/types.ts b/x-pack/plugins/aiops_api/common/types.ts index dd1cc5f9ef82b6..07eb3a34ece67f 100644 --- a/x-pack/plugins/aiops_api/common/types.ts +++ b/x-pack/plugins/aiops_api/common/types.ts @@ -5,8 +5,6 @@ * 2.0. */ -import type { SignificantItem } from '@kbn/ml-agg-utils'; - export interface GetAiopsLogRateAnalysisFunctionResponse { content: | string @@ -41,6 +39,5 @@ export interface GetAiopsLogRateAnalysisFunctionResponse { documentSamplingFactorForAnalysis?: number; extendedChangePoint: { startTs: number; endTs: number }; }; - significantItems: SignificantItem[]; }; } diff --git a/x-pack/plugins/aiops_api/kibana.jsonc b/x-pack/plugins/aiops_api/kibana.jsonc index d07b08cbf02e7d..e3defd7c8a7dee 100644 --- a/x-pack/plugins/aiops_api/kibana.jsonc +++ b/x-pack/plugins/aiops_api/kibana.jsonc @@ -8,12 +8,15 @@ "server": true, "browser": true, "requiredPlugins": [ + "charts", + "data", + "fieldFormats", "licensing", "observabilityAIAssistant" ], "optionalPlugins": [], "requiredBundles": [ - "charts" + "charts", ] } } diff --git a/x-pack/plugins/aiops_api/public/functions/log_rate_analysis.tsx b/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx similarity index 52% rename from x-pack/plugins/aiops_api/public/functions/log_rate_analysis.tsx rename to x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx index 578bbb194b9d99..1fd1947ad93a5b 100644 --- a/x-pack/plugins/aiops_api/public/functions/log_rate_analysis.tsx +++ b/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx @@ -7,89 +7,44 @@ import React from 'react'; -import { Axis, Chart, HistogramBarSeries, Position, ScaleType, Settings } from '@elastic/charts'; -import type { BarStyleAccessor } from '@elastic/charts/dist/chart_types/xy_chart/utils/specs'; import type { EuiBasicTableColumn } from '@elastic/eui'; import { EuiInMemoryTable, EuiCode, EuiSpacer, EuiText } from '@elastic/eui'; +import type { CoreStart } from '@kbn/core/public'; import { i18n } from '@kbn/i18n'; import type { RegisterRenderFunctionDefinition, RenderFunction, } from '@kbn/observability-ai-assistant-plugin/public'; -import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common'; - -import type { AiopsApiPluginStartDeps } from '../types'; +import { SimpleDocumentCountChart, NARROW_COLUMN_WIDTH } from '@kbn/aiops-components'; import type { GetAiopsLogRateAnalysisFunctionResponse } from '../../common/types'; -const LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR = 'orange'; -const SPEC_ID = 'document_count'; -const NARROW_COLUMN_WIDTH = '120px'; - -const overallSeriesName = i18n.translate( - 'xpack.aiops.dataGrid.field.documentCountChart.seriesLabel', - { - defaultMessage: 'document count', - } -); +import type { AiopsApiPluginStartDeps } from '../types'; export function registerLogRateAnalysisRenderFunction({ + coreStart, registerRenderFunction, pluginsStart, }: { + coreStart: CoreStart; registerRenderFunction: RegisterRenderFunctionDefinition; pluginsStart: AiopsApiPluginStartDeps; }) { - console.log('REGISTER FUNCTION'); const renderFunction: RenderFunction<{}, GetAiopsLogRateAnalysisFunctionResponse> = ({ arguments: args, response, }) => { - console.log('response', response); - if (typeof response.content === 'string') { return null; } - // CHART - - const adjustedChartPoints = Object.entries(response.data.dateHistogram).map( - ([time, value]) => ({ - time: Number(time), - value, - }) - ); - - // const xAxisFormatter = fieldFormats.deserialize({ id: 'date' }); - const useLegacyTimeAxis = false; - - const barStyle = { - rect: { - opacity: 1, - fill: LOG_RATE_ANALYSIS_HIGHLIGHT_COLOR, - }, - }; - - const extendedChangePoint = response.data.logRateChange.extendedChangePoint; - - // Used to highlight an auto-detected change point in the date histogram. - const barStyleAccessor: BarStyleAccessor = (d, g) => { - return g.specId === 'document_count' && - extendedChangePoint && - d.x > extendedChangePoint.startTs && - d.x < extendedChangePoint.endTs - ? barStyle - : null; - }; - - // TABLE - const tableItems = response.content.significantItems; const columns: Array> = [ { 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldName', + width: NARROW_COLUMN_WIDTH, field: 'field', name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldNameLabel', { defaultMessage: 'Field name', @@ -140,70 +95,38 @@ export function registerLogRateAnalysisRenderFunction({ }, ]; - const sorting = { - sort: { - field: 'logIncrease', - direction: 'desc' as const, - }, - }; - return (
- - - - xAxisFormatter.convert(value)} - labelFormat={useLegacyTimeAxis ? undefined : () => ''} - timeAxisLayerCount={useLegacyTimeAxis ? 0 : 2} - style={useLegacyTimeAxis ? {} : MULTILAYER_TIME_AXIS_STYLE} - /> - {adjustedChartPoints?.length && ( - - )} - + dateHistogram={response.data.dateHistogram} + changePoint={{ + ...response.data.logRateChange.extendedChangePoint, + key: 0, + type: 'spike', + }} + />

- More analysis options available in{' '} - Log Rate Analysis. + Showing the top 5 statistically significant log rate change contributors. A view with + all results is available in{' '} + Log Rate Analysis. The AI assistant + considers all results.

diff --git a/x-pack/plugins/aiops_api/public/functions/index.ts b/x-pack/plugins/aiops_api/public/functions/index.ts index 8ecc81491828c4..86e356cf63e61f 100644 --- a/x-pack/plugins/aiops_api/public/functions/index.ts +++ b/x-pack/plugins/aiops_api/public/functions/index.ts @@ -5,16 +5,19 @@ * 2.0. */ +import type { CoreStart } from '@kbn/core/public'; import type { RegisterRenderFunctionDefinition } from '@kbn/observability-ai-assistant-plugin/public/types'; import type { AiopsApiPluginStartDeps } from '../types'; -import { registerLogRateAnalysisRenderFunction } from './log_rate_analysis'; +import { registerLogRateAnalysisRenderFunction } from './get_aiops_log_rate_analysis'; export async function registerFunctions({ + coreStart, registerRenderFunction, pluginsStart, }: { + coreStart: CoreStart; registerRenderFunction: RegisterRenderFunctionDefinition; pluginsStart: AiopsApiPluginStartDeps; }) { - registerLogRateAnalysisRenderFunction({ pluginsStart, registerRenderFunction }); + registerLogRateAnalysisRenderFunction({ coreStart, pluginsStart, registerRenderFunction }); } diff --git a/x-pack/plugins/aiops_api/public/plugin.ts b/x-pack/plugins/aiops_api/public/plugin.ts index 39ac7e3c787c6d..56355f4db25a6a 100644 --- a/x-pack/plugins/aiops_api/public/plugin.ts +++ b/x-pack/plugins/aiops_api/public/plugin.ts @@ -5,7 +5,7 @@ * 2.0. */ -import type { CoreStart, Plugin } from '@kbn/core/public'; +import type { Plugin } from '@kbn/core/public'; import { type CoreSetup } from '@kbn/core/public'; import { firstValueFrom } from 'rxjs'; import type { @@ -28,24 +28,20 @@ export class AiopsApiPlugin { public setup(core: AiopsCoreSetup, { licensing }: AiopsApiPluginSetupDeps) { Promise.all([firstValueFrom(licensing.license$), core.getStartServices()]).then( - ([license, [coreStart, pluginStart]]) => { - if (license.hasAtLeast('platinum')) { - // Do something here + ([license, [coreStart, pluginsStart]]) => { + if (license.hasAtLeast('enterprise')) { + const service = pluginsStart.observabilityAIAssistant.service; + + service.register(async ({ registerRenderFunction }) => { + const { registerFunctions } = await import('./functions'); + + await registerFunctions({ coreStart, pluginsStart, registerRenderFunction }); + }); } } ); } - public start(core: CoreStart, pluginsStart: AiopsApiPluginStartDeps): AiopsApiPluginStart { - console.log('======== REGISTER!!!!'); - const service = pluginsStart.observabilityAIAssistant.service; - - service.register(async ({ registerRenderFunction }) => { - const { registerFunctions } = await import('./functions'); - - await registerFunctions({ pluginsStart, registerRenderFunction }); - }); - } - + public start() {} public stop() {} } diff --git a/x-pack/plugins/aiops_api/public/types.ts b/x-pack/plugins/aiops_api/public/types.ts index 83a956349f42d0..4087422af83e08 100644 --- a/x-pack/plugins/aiops_api/public/types.ts +++ b/x-pack/plugins/aiops_api/public/types.ts @@ -5,20 +5,16 @@ * 2.0. */ +import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { ChartsPluginStart } from '@kbn/charts-plugin/public'; import type { ExecutionContextStart } from '@kbn/core-execution-context-browser'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { SecurityPluginStart, SecurityPluginSetup } from '@kbn/security-plugin/public'; -import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public'; -import type { LensPublicStart } from '@kbn/lens-plugin/public'; import type { LicensingPluginSetup, LicensingPluginStart } from '@kbn/licensing-plugin/public'; -import type { SharePluginStart } from '@kbn/share-plugin/public'; -import type { UiActionsStart } from '@kbn/ui-actions-plugin/public'; import type { ObservabilityAIAssistantPublicSetup, ObservabilityAIAssistantPublicStart, } from '@kbn/observability-ai-assistant-plugin/public'; -import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/public'; export interface AiopsApiPluginSetupDeps { @@ -29,14 +25,10 @@ export interface AiopsApiPluginSetupDeps { export interface AiopsApiPluginStartDeps { licensing: LicensingPluginStart; - share: SharePluginStart; security: SecurityPluginStart; - lens: LensPublicStart; data: DataPublicPluginStart; charts: ChartsPluginStart; - uiActions: UiActionsStart; - unifiedSearch: UnifiedSearchPublicPluginStart; - storage: IStorageWrapper; + fieldFormats: FieldFormatsStart; executionContext: ExecutionContextStart; usageCollection: UsageCollectionSetup; observabilityAIAssistant: ObservabilityAIAssistantPublicStart; diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index dcecbe979baccc..304bc2a08980f6 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -109,7 +109,7 @@ export function registerGetAiopsLogRateAnalysisFunction({ const latestMs = dateMath.parse(args.end, { roundUp: true })?.valueOf(); if (earliestMs === undefined || latestMs === undefined) { - return { content: 'Could not parse time range.', data: { significantItems: [] } }; + return { content: 'Could not parse time range.', data: {} }; } const delta = latestMs - earliestMs; const dayMs = 86400 * 1000; @@ -338,11 +338,13 @@ export function registerGetAiopsLogRateAnalysisFunction({ type: type === 'keyword' ? 'metadata' : 'log message pattern', documentCount: doc_count, baselineCount: bg_count, + logIncreaseSort: bg_count > 0 ? doc_count / bg_count : doc_count, logIncrease: bg_count > 0 ? `${Math.round((doc_count / bg_count) * 100) / 100}x increase` - : `${doc_count} documents up from 0 documents in baseline`, - })), + : `${doc_count} docs up from 0 in baseline`, + })) + .sort((a, b) => b.logIncreaseSort - a.logIncreaseSort), }, data: { dateHistogram: buckets, @@ -359,7 +361,6 @@ export function registerGetAiopsLogRateAnalysisFunction({ }, })}`, logRateChange, - significantItems: [...significantTerms, ...significantCategories], }, }; } diff --git a/x-pack/plugins/aiops_api/server/types.ts b/x-pack/plugins/aiops_api/server/types.ts index 7dff5e466bfdfd..67f7f5ab5eb39a 100755 --- a/x-pack/plugins/aiops_api/server/types.ts +++ b/x-pack/plugins/aiops_api/server/types.ts @@ -9,10 +9,12 @@ import type { PluginSetup, PluginStart } from '@kbn/data-plugin/server'; import type { LicensingPluginSetup } from '@kbn/licensing-plugin/server'; import type { CasesServerSetup } from '@kbn/cases-plugin/server'; import type { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; +import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import type { ObservabilityAIAssistantServerSetup, ObservabilityAIAssistantServerStart, } from '@kbn/observability-ai-assistant-plugin/server'; +import type { IUiSettingsClient } from '@kbn/core/public'; export interface AiopsApiPluginSetupDeps { data: PluginSetup; @@ -24,6 +26,8 @@ export interface AiopsApiPluginSetupDeps { export interface AiopsApiPluginStartDeps { data: PluginStart; + fieldFormats: FieldFormatsStart; + uiSettings: IUiSettingsClient; observabilityAIAssistant: ObservabilityAIAssistantServerStart; } From 17f5a47a1717b2ee74bdea1daa3f9a76111bb50b Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 May 2024 12:50:51 +0200 Subject: [PATCH 16/31] aiops_components: consolidate --- x-pack/packages/ml/aiops_components/index.ts | 2 - .../src/document_count_chart/axis.tsx | 39 ++++++++++++++ .../src/document_count_chart/constants.ts | 1 + .../document_count_chart.tsx | 24 ++++----- .../src/document_count_chart/index.ts | 1 - .../simple_document_count_chart.tsx | 52 ++++++++----------- x-pack/plugins/aiops_api/public/plugin.ts | 13 +++-- 7 files changed, 80 insertions(+), 52 deletions(-) create mode 100644 x-pack/packages/ml/aiops_components/src/document_count_chart/axis.tsx diff --git a/x-pack/packages/ml/aiops_components/index.ts b/x-pack/packages/ml/aiops_components/index.ts index 1b032e17a755d4..8877abda267c88 100644 --- a/x-pack/packages/ml/aiops_components/index.ts +++ b/x-pack/packages/ml/aiops_components/index.ts @@ -8,11 +8,9 @@ export { DualBrush, DualBrushAnnotation } from './src/dual_brush'; export { ProgressControls } from './src/progress_controls'; export { - documentCountChartOverallSeriesName, DocumentCountChart, DocumentCountChartRedux, SimpleDocumentCountChart, - DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID, getLogRateAnalysisBarStyleAccessor, } from './src/document_count_chart'; export type { diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/axis.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/axis.tsx new file mode 100644 index 00000000000000..6f41ec3bed0dcf --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/axis.tsx @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FC } from 'react'; + +import { Axis, Position } from '@elastic/charts'; + +import type { FieldFormat } from '@kbn/field-formats-plugin/common'; +import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common'; + +export const DocumentCountChartAxisX: FC = () => ( + +); + +interface DocumentCountChartAxisYProps { + formatter: FieldFormat; + useLegacyTimeAxis: boolean; +} + +export const DocumentCountChartAxisY: FC = ({ + formatter, + useLegacyTimeAxis, +}) => { + return ( + formatter.convert(value)} + labelFormat={useLegacyTimeAxis ? undefined : () => ''} + timeAxisLayerCount={useLegacyTimeAxis ? 0 : 2} + style={useLegacyTimeAxis ? {} : MULTILAYER_TIME_AXIS_STYLE} + /> + ); +}; diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts b/x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts index 2260d0d6282bfc..aa7fb63889f22d 100644 --- a/x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/constants.ts @@ -5,4 +5,5 @@ * 2.0. */ +export const DOCUMENT_COUNT_CHART_DEFFAULT_HEIGHT = 120; export const DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID = 'document_count'; diff --git a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx index a16531f842005a..093f78100d8aab 100644 --- a/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx +++ b/x-pack/packages/ml/aiops_components/src/document_count_chart/document_count_chart.tsx @@ -14,7 +14,7 @@ import type { XYChartElementEvent, XYBrushEvent, } from '@elastic/charts'; -import { Axis, Chart, HistogramBarSeries, Position, ScaleType, Settings } from '@elastic/charts'; +import { Chart, HistogramBarSeries, ScaleType, Settings } from '@elastic/charts'; import type { BarStyleAccessor, RectAnnotationSpec, @@ -33,15 +33,18 @@ import { type WindowParameters, } from '@kbn/aiops-log-rate-analysis'; import { type BrushSelectionUpdatePayload } from '@kbn/aiops-log-rate-analysis/state'; -import { MULTILAYER_TIME_AXIS_STYLE } from '@kbn/charts-plugin/common'; import type { DataPublicPluginStart } from '@kbn/data-plugin/public'; import type { ChartsPluginStart } from '@kbn/charts-plugin/public'; import type { FieldFormatsStart } from '@kbn/field-formats-plugin/public'; import { DualBrush, DualBrushAnnotation } from '../..'; +import { DocumentCountChartAxisX, DocumentCountChartAxisY } from './axis'; import { BrushBadge } from './brush_badge'; -import { DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID } from './constants'; +import { + DOCUMENT_COUNT_CHART_DEFFAULT_HEIGHT, + DOCUMENT_COUNT_CHART_OVERALL_SERIES_SPEC_ID, +} from './constants'; import { documentCountChartOverallSeriesName, documentCountChartOverallSeriesNameWithSplit, @@ -465,7 +468,7 @@ export const DocumentCountChart: FC = (props) => { = (props) => { showLegend={false} locale={i18n.getLocale()} /> - - xAxisFormatter.convert(value)} - labelFormat={useLegacyTimeAxis ? undefined : () => ''} - timeAxisLayerCount={useLegacyTimeAxis ? 0 : 2} - style={useLegacyTimeAxis ? {} : MULTILAYER_TIME_AXIS_STYLE} + + {adjustedChartPoints?.length && ( = ({ const xAxisFormatter = fieldFormats.deserialize({ id: 'date' }); const useLegacyTimeAxis = uiSettings.get('visualization:useLegacyTimeAxis', false); + const timeZone = getTimeZone(uiSettings); // Used to highlight an auto-detected change point in the date histogram. const barStyleAccessor = getLogRateAnalysisBarStyleAccessor(changePoint); @@ -55,7 +60,7 @@ export const SimpleDocumentCountChart: FC = ({ = ({ showLegendExtra={false} locale={i18n.getLocale()} /> - - xAxisFormatter.convert(value)} - labelFormat={useLegacyTimeAxis ? undefined : () => ''} - timeAxisLayerCount={useLegacyTimeAxis ? 0 : 2} - style={useLegacyTimeAxis ? {} : MULTILAYER_TIME_AXIS_STYLE} + + + - {adjustedChartPoints?.length && ( - - )} ); }; diff --git a/x-pack/plugins/aiops_api/public/plugin.ts b/x-pack/plugins/aiops_api/public/plugin.ts index 56355f4db25a6a..a4b003f4b4a8f8 100644 --- a/x-pack/plugins/aiops_api/public/plugin.ts +++ b/x-pack/plugins/aiops_api/public/plugin.ts @@ -30,13 +30,12 @@ export class AiopsApiPlugin Promise.all([firstValueFrom(licensing.license$), core.getStartServices()]).then( ([license, [coreStart, pluginsStart]]) => { if (license.hasAtLeast('enterprise')) { - const service = pluginsStart.observabilityAIAssistant.service; - - service.register(async ({ registerRenderFunction }) => { - const { registerFunctions } = await import('./functions'); - - await registerFunctions({ coreStart, pluginsStart, registerRenderFunction }); - }); + pluginsStart.observabilityAIAssistant.service.register( + async ({ registerRenderFunction }) => { + const { registerFunctions } = await import('./functions'); + await registerFunctions({ coreStart, pluginsStart, registerRenderFunction }); + } + ); } } ); From 5d487e115420d01baa094ad9e7fa885054fdd11a Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 May 2024 13:00:03 +0200 Subject: [PATCH 17/31] aiops_components: simple analysis results table --- x-pack/packages/ml/aiops_components/index.ts | 11 +++ .../src/results_table/index.ts | 1 + .../simple_analysis_results_table.tsx | 96 +++++++++++++++++++ .../functions/get_aiops_log_rate_analysis.tsx | 72 +------------- 4 files changed, 111 insertions(+), 69 deletions(-) create mode 100644 x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx diff --git a/x-pack/packages/ml/aiops_components/index.ts b/x-pack/packages/ml/aiops_components/index.ts index 8877abda267c88..3d4f8039cc8995 100644 --- a/x-pack/packages/ml/aiops_components/index.ts +++ b/x-pack/packages/ml/aiops_components/index.ts @@ -18,3 +18,14 @@ export type { BrushSelectionUpdateHandler, DocumentCountChartProps, } from './src/document_count_chart'; +<<<<<<< HEAD +======= +export { + useLogRateAnalysisStateContext, + LogRateAnalysisStateProvider, + type GroupTableItem, + type GroupTableItemGroup, + type TableItemAction, +} from './src/log_rate_analysis_state_provider'; +export { SimpleAnalysisResultsTable, NARROW_COLUMN_WIDTH } from './src/results_table'; +>>>>>>> 49b1d08606e3 (aiops_components: simple analysis results table) diff --git a/x-pack/packages/ml/aiops_components/src/results_table/index.ts b/x-pack/packages/ml/aiops_components/src/results_table/index.ts index e4f592441a35ee..15f422281862a9 100644 --- a/x-pack/packages/ml/aiops_components/src/results_table/index.ts +++ b/x-pack/packages/ml/aiops_components/src/results_table/index.ts @@ -6,3 +6,4 @@ */ export { NARROW_COLUMN_WIDTH } from './constants'; +export { SimpleAnalysisResultsTable } from './simple_analysis_results_table'; diff --git a/x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx b/x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx new file mode 100644 index 00000000000000..c941694fdc3440 --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { type FC } from 'react'; + +import type { EuiBasicTableColumn } from '@elastic/eui'; +import { EuiInMemoryTable, EuiText, EuiCode } from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; + +import { NARROW_COLUMN_WIDTH } from './constants'; + +interface TableItem { + field: string; + value: string | number; + type: 'metadata' | 'log message pattern'; + documentCount: number; + baselineCount: number; + logIncrease: string; +} + +interface SimpleAnalysisResultsTableProps { + tableItems: TableItem[]; +} + +export const SimpleAnalysisResultsTable: FC = ({ tableItems }) => { + const columns: Array> = [ + { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldName', + width: NARROW_COLUMN_WIDTH, + field: 'field', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldNameLabel', { + defaultMessage: 'Field name', + }), + render: (_, { field }) => { + return ( + + {field} + + ); + }, + sortable: true, + valign: 'middle', + }, + { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldValue', + field: 'value', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldValueLabel', { + defaultMessage: 'Field value', + }), + render: (_, { value, type }) => ( + + {type === 'metadata' ? ( + String(value) + ) : ( + + + {String(value)} + + + )} + + ), + sortable: true, + textOnly: true, + truncateText: { lines: 3 }, + valign: 'middle', + }, + { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnDocCount', + width: NARROW_COLUMN_WIDTH, + field: 'logIncrease', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.docCountLabel', { + defaultMessage: 'Increase', + }), + sortable: true, + valign: 'middle', + }, + ]; + + return ( + + ); +}; diff --git a/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx b/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx index 1fd1947ad93a5b..52aa5ec8e02735 100644 --- a/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx +++ b/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx @@ -7,16 +7,14 @@ import React from 'react'; -import type { EuiBasicTableColumn } from '@elastic/eui'; -import { EuiInMemoryTable, EuiCode, EuiSpacer, EuiText } from '@elastic/eui'; +import { EuiSpacer } from '@elastic/eui'; import type { CoreStart } from '@kbn/core/public'; -import { i18n } from '@kbn/i18n'; import type { RegisterRenderFunctionDefinition, RenderFunction, } from '@kbn/observability-ai-assistant-plugin/public'; -import { SimpleDocumentCountChart, NARROW_COLUMN_WIDTH } from '@kbn/aiops-components'; +import { SimpleDocumentCountChart, SimpleAnalysisResultsTable } from '@kbn/aiops-components'; import type { GetAiopsLogRateAnalysisFunctionResponse } from '../../common/types'; @@ -39,62 +37,6 @@ export function registerLogRateAnalysisRenderFunction({ return null; } - const tableItems = response.content.significantItems; - - const columns: Array> = [ - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldName', - width: NARROW_COLUMN_WIDTH, - field: 'field', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldNameLabel', { - defaultMessage: 'Field name', - }), - render: (_, { field }) => { - return ( - - {field} - - ); - }, - sortable: true, - valign: 'middle', - }, - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldValue', - field: 'value', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldValueLabel', { - defaultMessage: 'Field value', - }), - render: (_, { value, type }) => ( - - {type === 'metadata' ? ( - String(value) - ) : ( - - - {String(value)} - - - )} - - ), - sortable: true, - textOnly: true, - truncateText: { lines: 3 }, - valign: 'middle', - }, - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnDocCount', - width: NARROW_COLUMN_WIDTH, - field: 'logIncrease', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.docCountLabel', { - defaultMessage: 'Increase', - }), - sortable: true, - valign: 'middle', - }, - ]; - return (
- +

From 52299505c1b06f30eb3835894eba30ba08830f2b Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 May 2024 13:29:15 +0200 Subject: [PATCH 18/31] aiops_components: consolidate table column code --- x-pack/packages/ml/agg_utils/src/types.ts | 3 + x-pack/packages/ml/aiops_components/index.ts | 14 +--- .../src/results_table/columns.tsx | 78 ++++++++++++++++++ .../src/results_table/index.ts | 1 + .../simple_analysis_results_table.tsx | 81 ++++++------------- .../log_rate_analysis_results_table.tsx | 3 +- .../use_columns.tsx | 37 +-------- 7 files changed, 116 insertions(+), 101 deletions(-) create mode 100644 x-pack/packages/ml/aiops_components/src/results_table/columns.tsx diff --git a/x-pack/packages/ml/agg_utils/src/types.ts b/x-pack/packages/ml/agg_utils/src/types.ts index a60648c10f1d91..1982e38dba9f29 100644 --- a/x-pack/packages/ml/agg_utils/src/types.ts +++ b/x-pack/packages/ml/agg_utils/src/types.ts @@ -161,6 +161,9 @@ export interface SignificantItem extends FieldValuePair { /** The normalized score for the significant item. */ normalizedScore: number; + /** A human readable description of the log rate change. */ + changeDescription?: string; + /** An optional histogram for the significant item. */ histogram?: SignificantItemHistogramItem[]; diff --git a/x-pack/packages/ml/aiops_components/index.ts b/x-pack/packages/ml/aiops_components/index.ts index 3d4f8039cc8995..efa34d94cdaa44 100644 --- a/x-pack/packages/ml/aiops_components/index.ts +++ b/x-pack/packages/ml/aiops_components/index.ts @@ -18,14 +18,8 @@ export type { BrushSelectionUpdateHandler, DocumentCountChartProps, } from './src/document_count_chart'; -<<<<<<< HEAD -======= export { - useLogRateAnalysisStateContext, - LogRateAnalysisStateProvider, - type GroupTableItem, - type GroupTableItemGroup, - type TableItemAction, -} from './src/log_rate_analysis_state_provider'; -export { SimpleAnalysisResultsTable, NARROW_COLUMN_WIDTH } from './src/results_table'; ->>>>>>> 49b1d08606e3 (aiops_components: simple analysis results table) + getFieldValueColumn, + SimpleAnalysisResultsTable, + NARROW_COLUMN_WIDTH, +} from './src/results_table'; diff --git a/x-pack/packages/ml/aiops_components/src/results_table/columns.tsx b/x-pack/packages/ml/aiops_components/src/results_table/columns.tsx new file mode 100644 index 00000000000000..49c29b6cd5445c --- /dev/null +++ b/x-pack/packages/ml/aiops_components/src/results_table/columns.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; + +import type { EuiBasicTableColumn } from '@elastic/eui'; +import { EuiCode, EuiText } from '@elastic/eui'; + +import { i18n } from '@kbn/i18n'; +import { type SignificantItem } from '@kbn/ml-agg-utils'; + +import { NARROW_COLUMN_WIDTH } from './constants'; + +const TRUNCATE_TEXT_LINES = 3; + +export const getFieldNameColumn = (): EuiBasicTableColumn => { + return { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldName', + width: NARROW_COLUMN_WIDTH, + field: 'fieldName', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldNameLabel', { + defaultMessage: 'Field name', + }), + render: (_, { fieldName }) => { + return ( + + {fieldName} + + ); + }, + sortable: true, + valign: 'middle', + }; +}; + +export const getFieldValueColumn = (narrow = false): EuiBasicTableColumn => { + return { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldValue', + field: 'fieldValue', + width: narrow ? '17%' : '25%', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldValueLabel', { + defaultMessage: 'Field value', + }), + render: (_, { fieldValue, type }) => ( + + {type === 'keyword' ? ( + String(fieldValue) + ) : ( + + + {String(fieldValue)} + + + )} + + ), + sortable: true, + textOnly: true, + truncateText: { lines: TRUNCATE_TEXT_LINES }, + valign: 'middle', + }; +}; + +export const getChangeDescriptionColumn = (): EuiBasicTableColumn => { + return { + 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnChangeDescription', + field: 'changeDescription', + name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.changeDescriptionLabel', { + defaultMessage: 'Increase', + }), + sortable: true, + valign: 'middle', + }; +}; diff --git a/x-pack/packages/ml/aiops_components/src/results_table/index.ts b/x-pack/packages/ml/aiops_components/src/results_table/index.ts index 15f422281862a9..db533515125317 100644 --- a/x-pack/packages/ml/aiops_components/src/results_table/index.ts +++ b/x-pack/packages/ml/aiops_components/src/results_table/index.ts @@ -6,4 +6,5 @@ */ export { NARROW_COLUMN_WIDTH } from './constants'; +export { getFieldValueColumn } from './columns'; export { SimpleAnalysisResultsTable } from './simple_analysis_results_table'; diff --git a/x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx b/x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx index c941694fdc3440..bea994127f9a74 100644 --- a/x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx +++ b/x-pack/packages/ml/aiops_components/src/results_table/simple_analysis_results_table.tsx @@ -8,11 +8,11 @@ import React, { type FC } from 'react'; import type { EuiBasicTableColumn } from '@elastic/eui'; -import { EuiInMemoryTable, EuiText, EuiCode } from '@elastic/eui'; +import { EuiInMemoryTable } from '@elastic/eui'; -import { i18n } from '@kbn/i18n'; +import { type SignificantItem } from '@kbn/ml-agg-utils'; -import { NARROW_COLUMN_WIDTH } from './constants'; +import { getFieldNameColumn, getFieldValueColumn, getChangeDescriptionColumn } from './columns'; interface TableItem { field: string; @@ -28,58 +28,27 @@ interface SimpleAnalysisResultsTableProps { } export const SimpleAnalysisResultsTable: FC = ({ tableItems }) => { - const columns: Array> = [ - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldName', - width: NARROW_COLUMN_WIDTH, - field: 'field', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldNameLabel', { - defaultMessage: 'Field name', - }), - render: (_, { field }) => { - return ( - - {field} - - ); - }, - sortable: true, - valign: 'middle', - }, - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldValue', - field: 'value', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldValueLabel', { - defaultMessage: 'Field value', - }), - render: (_, { value, type }) => ( - - {type === 'metadata' ? ( - String(value) - ) : ( - - - {String(value)} - - - )} - - ), - sortable: true, - textOnly: true, - truncateText: { lines: 3 }, - valign: 'middle', - }, - { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnDocCount', - width: NARROW_COLUMN_WIDTH, - field: 'logIncrease', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.docCountLabel', { - defaultMessage: 'Increase', - }), - sortable: true, - valign: 'middle', - }, + const adjustedTableItems: SignificantItem[] = tableItems.map((item) => { + return { + key: `${item.field}:${item.value}`, + fieldName: item.field, + fieldValue: item.value, + doc_count: item.documentCount, + bg_count: item.baselineCount, + total_doc_count: 0, + total_bg_count: 0, + score: 0, + pValue: 0, + normalizedScore: 0, + type: item.type === 'metadata' ? 'keyword' : 'log_pattern', + changeDescription: item.logIncrease, + }; + }); + + const columns: Array> = [ + getFieldNameColumn(), + getFieldValueColumn(), + getChangeDescriptionColumn(), ]; return ( @@ -87,7 +56,7 @@ export const SimpleAnalysisResultsTable: FC = ( data-test-subj="aiopsLogRateAnalysisResultsTable" compressed columns={columns} - items={tableItems.splice(0, 5)} + items={adjustedTableItems.splice(0, 5)} loading={false} sorting={false} pagination={false} diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx index 38cf611b4bdd1b..dc3b995e30f6b7 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/log_rate_analysis_results_table.tsx @@ -20,9 +20,10 @@ import { useAppDispatch, useAppSelector, } from '@kbn/aiops-log-rate-analysis/state'; - import type { GroupTableItemGroup } from '@kbn/aiops-log-rate-analysis/state'; + import { useEuiTheme } from '../../hooks/use_eui_theme'; + import { useColumns, LOG_RATE_ANALYSIS_RESULTS_TABLE_TYPE } from './use_columns'; const PAGINATION_SIZE_OPTIONS = [5, 10, 20, 50]; diff --git a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_columns.tsx b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_columns.tsx index 40ea76fceddc41..0ae6aa851c5385 100644 --- a/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_columns.tsx +++ b/x-pack/plugins/aiops/public/components/log_rate_analysis_results_table/use_columns.tsx @@ -6,14 +6,7 @@ */ import React, { useMemo } from 'react'; -import { - type EuiBasicTableColumn, - EuiBadge, - EuiCode, - EuiIcon, - EuiIconTip, - EuiText, -} from '@elastic/eui'; +import { type EuiBasicTableColumn, EuiBadge, EuiIcon, EuiIconTip } from '@elastic/eui'; import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n-react'; @@ -22,6 +15,7 @@ import { getCategoryQuery } from '@kbn/aiops-log-pattern-analysis/get_category_q import type { FieldStatsServices } from '@kbn/unified-field-list/src/components/field_stats'; import { useAppSelector } from '@kbn/aiops-log-rate-analysis/state'; import { LOG_RATE_ANALYSIS_TYPE } from '@kbn/aiops-log-rate-analysis'; +import { getFieldValueColumn } from '@kbn/aiops-components'; import { getFailedTransactionsCorrelationImpactLabel } from './get_failed_transactions_correlation_impact_label'; import { FieldStatsPopover } from '../field_stats_popover'; import { useAiopsAppContext } from '../../hooks/use_aiops_app_context'; @@ -33,7 +27,6 @@ import { useCopyToClipboardAction } from './use_copy_to_clipboard_action'; import { MiniHistogram } from '../mini_histogram'; import { getBaselineAndDeviationRates, getLogRateChange } from './get_baseline_and_deviation_rates'; -const TRUNCATE_TEXT_LINES = 3; const UNIQUE_COLUMN_WIDTH = '40px'; const NOT_AVAILABLE = '--'; @@ -255,31 +248,7 @@ export const useColumns = ( sortable: true, valign: 'middle', }, - ['Field value']: { - 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnFieldValue', - field: 'fieldValue', - width: skippedColumns.length < 3 ? '17%' : '25%', - name: i18n.translate('xpack.aiops.logRateAnalysis.resultsTable.fieldValueLabel', { - defaultMessage: 'Field value', - }), - render: (_, { fieldValue, type }) => ( - - {type === 'keyword' ? ( - String(fieldValue) - ) : ( - - - {String(fieldValue)} - - - )} - - ), - sortable: true, - textOnly: true, - truncateText: { lines: TRUNCATE_TEXT_LINES }, - valign: 'middle', - }, + ['Field value']: getFieldValueColumn(skippedColumns.length < 3), ['Log rate']: { 'data-test-subj': 'aiopsLogRateAnalysisResultsTableColumnLogRate', width: '8%', From 8bedb018ae3e15d4cf4afc2613d8e0be7d600727 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 14 May 2024 17:47:40 +0200 Subject: [PATCH 19/31] support for dips --- .../src/results_table/columns.tsx | 2 +- .../simple_analysis_results_table.tsx | 4 +- .../get_extended_change_point.ts | 9 +- .../functions/get_aiops_log_rate_analysis.tsx | 4 +- .../get_aiops_log_rate_analysis_function.ts | 121 +++++++++++------- 5 files changed, 86 insertions(+), 54 deletions(-) diff --git a/x-pack/packages/ml/aiops_components/src/results_table/columns.tsx b/x-pack/packages/ml/aiops_components/src/results_table/columns.tsx index 49c29b6cd5445c..e3068b2d25634e 100644 --- a/x-pack/packages/ml/aiops_components/src/results_table/columns.tsx +++ b/x-pack/packages/ml/aiops_components/src/results_table/columns.tsx @@ -70,7 +70,7 @@ export const getChangeDescriptionColumn = (): EuiBasicTableColumn = ( pValue: 0, normalizedScore: 0, type: item.type === 'metadata' ? 'keyword' : 'log_pattern', - changeDescription: item.logIncrease, + changeDescription: item.logRateChange, }; }); diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/get_extended_change_point.ts b/x-pack/packages/ml/aiops_log_rate_analysis/get_extended_change_point.ts index c0e990dea705a9..6b7f6a5154b8ae 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/get_extended_change_point.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/get_extended_change_point.ts @@ -27,18 +27,21 @@ export const getExtendedChangePoint = (buckets: Record, changePo let uIndex = cpIndex + 1; while ( - lIndex >= 0 && + lIndex > 0 && Math.abs(bucketValues[lIndex] - meanValue) > Math.abs(bucketValues[lIndex] - cpValue) ) { lIndex--; } while ( - uIndex < bucketValues.length && + uIndex < bucketValues.length - 1 && Math.abs(bucketValues[uIndex] - meanValue) > Math.abs(bucketValues[uIndex] - cpValue) ) { uIndex++; } - return { startTs: +bucketKeys[lIndex], endTs: +bucketKeys[uIndex] }; + const startTs = +bucketKeys[lIndex]; + const endTs = +bucketKeys[uIndex]; + + return { startTs, endTs }; }; diff --git a/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx b/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx index 52aa5ec8e02735..4ccbd313f71fd7 100644 --- a/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx +++ b/x-pack/plugins/aiops_api/public/functions/get_aiops_log_rate_analysis.tsx @@ -34,7 +34,7 @@ export function registerLogRateAnalysisRenderFunction({ response, }) => { if (typeof response.content === 'string') { - return null; + return

{response.content}

; } return ( @@ -53,7 +53,7 @@ export function registerLogRateAnalysisRenderFunction({ }} /> - +

diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index 304bc2a08980f6..d936d32a6023d0 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -84,23 +84,20 @@ export function registerGetAiopsLogRateAnalysisFunction({ } ), description: dedent(` - Log rate analysis is an Elastic AIOps feature to identify causes of spike/dips in time series of log rates. The analysis returns significant field/value pairs found in the log rate change sorted by logIncrease descending. + Log rate analysis is an AIOps feature to identify causes of spike/dips in log rate time series. If a spike gets detected, the significant items are the log patterns that are more frequent in the spike compared to the baseline. If a dip gets detected, the significant items are the log patterns that are more frequent in the baseline and are reduced or missing in the dip. - Users will see a UI with a chart of the log rate over time, with the log rate change highlighted. The UI will also include a table with all identified significant field/value pairs. + Your task is the following: - Your task is to summarize the provided data and provide context about the given data on display in the UI: - - - Infer and explain the environment the data was obtained from, summarize the most contributing field/value pairs and provide some field/value pair examples. - - If the data hints at a security or performance issue, explain the root cause and 1-2 steps to remediate the problem. - - Your output should be very concise, non-repeating, to-the-point. Your audience are technical users like SREs or Security Analysts using Elasticsearch and Kibana. + - Briefly infer the environment based on all data available and summarize field/value pairs with up to 3 individual examples. Summary hint: Items with the same logRateChange might be related. + - Evaluate if the log rate change could be results of regular operations or if the data hints at a security or performance issue, if applicable briefly explain the root cause with concrete steps to remediate the problem. + - Your output should be brief and very concise. Your audience are technical users like SREs or Security Analysts using Elasticsearch and Kibana. + - Limit overall output to 300 words. `), parameters, }, async ({ arguments: args }, abortSignal): Promise => { const debugStartTime = Date.now(); const { esClient } = resources; - console.log('args', args); // CHANGE POINT DETECTION @@ -112,9 +109,25 @@ export function registerGetAiopsLogRateAnalysisFunction({ return { content: 'Could not parse time range.', data: {} }; } const delta = latestMs - earliestMs; + const dayMs = 86400 * 1000; - const threshold = dayMs * 22; - const intervalMs = delta > threshold ? dayMs : Math.round(delta / barTarget); + const dayThreshold = dayMs * 22; + + const weekMs = dayMs * 7; + const weekThreshold = weekMs * 22; + + const monthMs = dayMs * 30; + const monthThreshold = monthMs * 22; + + let intervalMs = Math.round(delta / barTarget); + + if (delta > monthThreshold) { + intervalMs = monthMs; + } else if (delta > weekThreshold) { + intervalMs = weekMs; + } else if (delta > dayThreshold) { + intervalMs = dayMs; + } const aggs: Record = { eventRate: { @@ -167,9 +180,6 @@ export function registerGetAiopsLogRateAnalysisFunction({ } ); - console.log('eventRate', histogram.aggregations.eventRate); - console.log('change_point_request', histogram.aggregations.change_point_request); - if (histogram.aggregations.change_point_request.bucket === undefined) { return { content: 'No log rate change detected.', data: [] }; } @@ -188,17 +198,12 @@ export function registerGetAiopsLogRateAnalysisFunction({ } const extendedChangePoint = getExtendedChangePoint(buckets, changePointTs); - console.log('extendedChangePoint', extendedChangePoint); - const logRateType = Object.keys(histogram.aggregations.change_point_request.type)[0]; - const logRateAttributeName = `${logRateType}LogRate`; + let logRateType = Object.keys(histogram.aggregations.change_point_request.type)[0]; // FIELD CANDIDATES const fieldCandidates: string[] = []; - let fieldCandidatesCount = fieldCandidates.length; const textFieldCandidates: string[] = []; - let totalDocCount = 0; - let zeroDocsFallback = false; const changePoint = { ...extendedChangePoint, key: changePointTs, @@ -212,7 +217,7 @@ export function registerGetAiopsLogRateAnalysisFunction({ changePoint ); - const params: AiopsLogRateAnalysisSchema = { + const indexInfoParams: AiopsLogRateAnalysisSchema = { index: args.index, start: earliestMs, end: latestMs, @@ -223,26 +228,43 @@ export function registerGetAiopsLogRateAnalysisFunction({ const indexInfo = await fetchIndexInfo( esClient, - params, + indexInfoParams, ['message', 'error.message'], abortSignal ); - console.log('indexInfo', indexInfo); + + const baselineNumBuckets = (wp.baselineMax - wp.baselineMin) / intervalMs; + const baselinePerBucket = indexInfo.baselineTotalDocCount / baselineNumBuckets; + + const deviationNumBuckets = (wp.deviationMax - wp.deviationMin) / intervalMs; + const deviationPerBucket = indexInfo.deviationTotalDocCount / deviationNumBuckets; + + const analysisWindowParameters = + deviationPerBucket > baselinePerBucket + ? wp + : { + baselineMin: wp.deviationMin, + baselineMax: wp.deviationMax, + deviationMin: wp.baselineMin, + deviationMax: wp.baselineMax, + }; + + logRateType = deviationPerBucket > baselinePerBucket ? 'spike' : 'dip'; + const logRateAttributeName = `${logRateType}LogRate`; + + const params: AiopsLogRateAnalysisSchema = { + ...indexInfoParams, + ...analysisWindowParameters, + }; fieldCandidates.push(...indexInfo.fieldCandidates); - fieldCandidatesCount = fieldCandidates.length; textFieldCandidates.push(...indexInfo.textFieldCandidates); - totalDocCount = indexInfo.deviationTotalDocCount; - zeroDocsFallback = indexInfo.zeroDocsFallback; const sampleProbability = getSampleProbability( indexInfo.deviationTotalDocCount + indexInfo.baselineTotalDocCount ); - console.log('sampleProbability', sampleProbability); // SIGNIFICANT ITEMS - fieldCandidatesCount = fieldCandidates.length; - // This will store the combined count of detected significant log patterns and keywords let fieldValuePairsCount = 0; @@ -315,6 +337,28 @@ export function registerGetAiopsLogRateAnalysisFunction({ extendedChangePoint, }; + const significantItems = [...significantTerms, ...significantCategories] + .filter(({ bg_count, doc_count }) => { + return doc_count > bg_count; + }) + .map(({ fieldName, fieldValue, type, doc_count, bg_count }) => ({ + field: fieldName, + value: fieldValue, + type: type === 'keyword' ? 'metadata' : 'log message pattern', + documentCount: doc_count, + baselineCount: bg_count, + logRateChangeSort: bg_count > 0 ? doc_count / bg_count : doc_count, + logRateChange: + bg_count > 0 + ? logRateType === 'spike' + ? `${Math.round((doc_count / bg_count) * 100) / 100}x increase` + : `${Math.round((bg_count / doc_count) * 100) / 100}x decrease` + : logRateType === 'spike' + ? `${doc_count} docs up from 0 in baseline` + : `0 docs down from ${doc_count} in baseline`, + })) + .sort((a, b) => b.logRateChangeSort - a.logRateChangeSort); + return { content: { logRateChange: { @@ -328,23 +372,7 @@ export function registerGetAiopsLogRateAnalysisFunction({ ? { documentSamplingFactorForAnalysis: sampleProbability } : {}), }, - significantItems: [...significantTerms, ...significantCategories] - .filter(({ bg_count, doc_count }) => { - return doc_count > bg_count; - }) - .map(({ fieldName, fieldValue, type, doc_count, bg_count }) => ({ - field: fieldName, - value: fieldValue, - type: type === 'keyword' ? 'metadata' : 'log message pattern', - documentCount: doc_count, - baselineCount: bg_count, - logIncreaseSort: bg_count > 0 ? doc_count / bg_count : doc_count, - logIncrease: - bg_count > 0 - ? `${Math.round((doc_count / bg_count) * 100) / 100}x increase` - : `${doc_count} docs up from 0 in baseline`, - })) - .sort((a, b) => b.logIncreaseSort - a.logIncreaseSort), + significantItems: significantItems.slice(0, 100), }, data: { dateHistogram: buckets, @@ -361,6 +389,7 @@ export function registerGetAiopsLogRateAnalysisFunction({ }, })}`, logRateChange, + significantItems: significantItems.slice(0, 100), }, }; } From b715f40557d9d6cc20ffa0435387201b205e9fa4 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 22 May 2024 10:44:17 +0200 Subject: [PATCH 20/31] fetch_simple_log_rate_analysis --- .../queries/fetch_change_point_detection.ts | 165 ++++++++++ .../queries/fetch_simple_log_rate_analysis.ts | 295 +++++++++++++++++ x-pack/plugins/aiops_api/common/types.ts | 43 +-- .../get_aiops_log_rate_analysis_function.ts | 305 +----------------- 4 files changed, 490 insertions(+), 318 deletions(-) create mode 100644 x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts create mode 100644 x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts new file mode 100644 index 00000000000000..bd8cf603de556d --- /dev/null +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts @@ -0,0 +1,165 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import dateMath from '@kbn/datemath'; +import type { ElasticsearchClient } from '@kbn/core/server'; + +import { getExtendedChangePoint } from '../get_extended_change_point'; +import { getWindowParametersForTrigger } from '../get_window_parameters_for_trigger'; +import type { DocumentCountStatsChangePoint } from '../types'; +import type { WindowParameters } from '../window_parameters'; + +// Change point detection requires a minimum of 22 buckets to be able to run. +const CHANGE_POINT_MIN_BUCKETS = 22; + +interface ChangePointDetectionData { + changePoint: DocumentCountStatsChangePoint; + changePointDocCount: number; + dateHistogramBuckets: Record; + intervalMs: number; + windowParameters: WindowParameters; +} +type ChangePointDetectionError = [string, null]; +type ChangePointDetectionSuccess = [null, ChangePointDetectionData]; +type ChangePointDetectionResponse = ChangePointDetectionError | ChangePointDetectionSuccess; + +/** + * Fetches change points for doc count histogram. + * + * @param esClient Elasticsearch client. + * @param index The Elasticsearch source index pattern. + * @param start The start of the time range, in Elasticsearch date math, like `now`. + * @param end The end of the time range, in Elasticsearch date math, like `now-24h`. + * @param timefield The Elasticesarch source index pattern time field. + * @param abortSignal Abort signal. + * @returns change point data. + */ +export const fetchChangePointDetection = async ( + esClient: ElasticsearchClient, + index: string, + earliestMs: number, + latestMs: number, + timefield: string, + searchQuery: estypes.QueryDslQueryContainer, + abortSignal: AbortSignal +): Promise => { + const barTarget = 75; + + const delta = latestMs - earliestMs; + + const dayMs = 86400 * 1000; + const dayThreshold = dayMs * CHANGE_POINT_MIN_BUCKETS; + + const weekMs = dayMs * 7; + const weekThreshold = weekMs * CHANGE_POINT_MIN_BUCKETS; + + const monthMs = dayMs * 30; + const monthThreshold = monthMs * CHANGE_POINT_MIN_BUCKETS; + + let intervalMs = Math.round(delta / barTarget); + + if (delta > monthThreshold) { + intervalMs = monthMs; + } else if (delta > weekThreshold) { + intervalMs = weekMs; + } else if (delta > dayThreshold) { + intervalMs = dayMs; + } + + const aggs: Record = { + eventRate: { + date_histogram: { + field: timefield, + fixed_interval: `${intervalMs}ms`, + min_doc_count: 0, + ...(earliestMs !== undefined && latestMs !== undefined + ? { + extended_bounds: { + min: earliestMs, + max: latestMs, + }, + } + : {}), + }, + }, + change_point_request: { + // @ts-expect-error missing from ES spec + change_point: { + buckets_path: 'eventRate>_count', + }, + }, + }; + + const searchBody: estypes.MsearchMultisearchBody = { + query: searchQuery, + aggs, + track_total_hits: false, + size: 0, + }; + + const histogram = await esClient.search( + { + index, + body: searchBody, + }, + { + signal: abortSignal, + maxRetries: 0, + } + ); + + if (histogram.aggregations === undefined) { + return ['No log rate change detected.', null]; + } + + if (histogram.aggregations.change_point_request.bucket === undefined) { + return ['No log rate change detected.', null]; + } + + const dateHistogramBuckets = histogram.aggregations.eventRate.buckets.reduce((acc, cur) => { + acc[cur.key] = cur.doc_count; + return acc; + }, {}); + + const changePointTs = dateMath + .parse(histogram.aggregations.change_point_request.bucket.key) + ?.valueOf(); + + if (changePointTs === undefined) { + return ['There was an error parsing the log rate change timestamp.', null]; + } + + const extendedChangePoint = getExtendedChangePoint(dateHistogramBuckets, changePointTs); + const logRateType = Object.keys(histogram.aggregations.change_point_request.type)[0]; + + const changePoint = { + ...extendedChangePoint, + key: changePointTs, + type: logRateType, + }; + + const windowParameters = getWindowParametersForTrigger( + extendedChangePoint.startTs, + intervalMs, + earliestMs, + latestMs, + changePoint + ); + + return [ + null, + { + changePoint: { ...extendedChangePoint, key: changePointTs, type: logRateType }, + changePointDocCount: histogram.aggregations.change_point_request.bucket.doc_count, + dateHistogramBuckets, + intervalMs, + windowParameters, + }, + ]; +}; diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts new file mode 100644 index 00000000000000..b0d9fe41a37052 --- /dev/null +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts @@ -0,0 +1,295 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { queue } from 'async'; +import { mean } from 'd3-array'; +import moment from 'moment'; + +import dateMath from '@kbn/datemath'; +import type { ElasticsearchClient } from '@kbn/core/server'; +import type { SignificantItem } from '@kbn/ml-agg-utils'; +import { getSampleProbability } from '@kbn/ml-random-sampler-utils'; +import { isPopulatedObject } from '@kbn/ml-is-populated-object'; + +import type { AiopsLogRateAnalysisSchema } from '../api/schema'; + +import { fetchChangePointDetection } from './fetch_change_point_detection'; +import { fetchIndexInfo } from './fetch_index_info'; +import { fetchSignificantCategories } from './fetch_significant_categories'; +import { fetchSignificantTermPValues } from './fetch_significant_term_p_values'; + +// Don't use more than 5 here otherwise Kibana will emit an error +// regarding a limit of abort signal listeners of more than 10. +const MAX_CONCURRENT_QUERIES = 5; + +interface FieldCandidate { + fieldCandidate: string; +} +const isFieldCandidate = (d: unknown): d is FieldCandidate => + isPopulatedObject(d, ['fieldCandidate']); + +interface TextFieldCandidate { + textFieldCandidate: string; +} +const isTextFieldCandidate = (d: unknown): d is FieldCandidate => + isPopulatedObject(d, ['textFieldCandidate']); + +type Candidate = FieldCandidate | TextFieldCandidate; + +export interface LogRateChange { + type: string; + timestamp: number; + logRateChangeCount: number; + averageLogRateCount: number; + logRateAggregationIntervalUsedForAnalysis: string; + documentSamplingFactorForAnalysis?: number; + extendedChangePoint: { startTs: number; endTs: number }; +} + +export interface SimpleSignificantItem { + field: string; + value: string | number; + type: 'metadata' | 'log message pattern'; + documentCount: number; + baselineCount: number; + logRateChangeSort: number; + logRateChange: string; +} + +/** + * Fetches log rate analysis data. + * + * @param esClient Elasticsearch client. + * @param index The Elasticsearch source index pattern. + * @param start The start of the time range, in Elasticsearch date math, like `now`. + * @param end The end of the time range, in Elasticsearch date math, like `now-24h`. + * @param timefield The Elasticesarch source index pattern time field. + * @param abortSignal Abort signal. + * @returns Log rate analysis data. + */ +export const fetchSimpleLogRateAnalysis = async ( + esClient: ElasticsearchClient, + index: string, + start: string, + end: string, + timefield: string, + abortSignal: AbortSignal +) => { + const debugStartTime = Date.now(); + + const earliestMs = dateMath.parse(start)?.valueOf(); + const latestMs = dateMath.parse(end, { roundUp: true })?.valueOf(); + + if (earliestMs === undefined || latestMs === undefined) { + throw new Error('Could not parse time range'); + } + + const searchQuery = { + range: { + [timefield]: { + gte: earliestMs, + lte: latestMs, + format: 'epoch_millis', + }, + }, + }; + + // CHANGE POINT DETECTION + const [error, resp] = await fetchChangePointDetection( + esClient, + index, + earliestMs, + latestMs, + timefield, + searchQuery, + abortSignal + ); + + if (error !== null) { + throw new Error(error); + } + + const { changePoint, changePointDocCount, dateHistogramBuckets, intervalMs, windowParameters } = + resp; + + // FIELD CANDIDATES + + const fieldCandidates: string[] = []; + const textFieldCandidates: string[] = []; + + const indexInfoParams: AiopsLogRateAnalysisSchema = { + index, + start: earliestMs, + end: latestMs, + searchQuery: JSON.stringify(searchQuery), + timeFieldName: timefield, + ...windowParameters, + }; + + const indexInfo = await fetchIndexInfo( + esClient, + indexInfoParams, + ['message', 'error.message'], + abortSignal + ); + + const baselineNumBuckets = + (windowParameters.baselineMax - windowParameters.baselineMin) / intervalMs; + const baselinePerBucket = indexInfo.baselineTotalDocCount / baselineNumBuckets; + + const deviationNumBuckets = + (windowParameters.deviationMax - windowParameters.deviationMin) / intervalMs; + const deviationPerBucket = indexInfo.deviationTotalDocCount / deviationNumBuckets; + + const analysisWindowParameters = + deviationPerBucket > baselinePerBucket + ? windowParameters + : { + baselineMin: windowParameters.deviationMin, + baselineMax: windowParameters.deviationMax, + deviationMin: windowParameters.baselineMin, + deviationMax: windowParameters.baselineMax, + }; + + const logRateType = deviationPerBucket > baselinePerBucket ? 'spike' : 'dip'; + + const params: AiopsLogRateAnalysisSchema = { + ...indexInfoParams, + ...analysisWindowParameters, + }; + + fieldCandidates.push(...indexInfo.fieldCandidates); + textFieldCandidates.push(...indexInfo.textFieldCandidates); + const sampleProbability = getSampleProbability( + indexInfo.deviationTotalDocCount + indexInfo.baselineTotalDocCount + ); + + // SIGNIFICANT ITEMS + + // This will store the combined count of detected significant log patterns and keywords + let fieldValuePairsCount = 0; + + const significantCategories: SignificantItem[] = []; + const significantTerms: SignificantItem[] = []; + const fieldsToSample = new Set(); + + const pValuesQueue = queue(async function (payload: Candidate) { + if (isFieldCandidate(payload)) { + const { fieldCandidate } = payload; + let pValues: Awaited> = []; + + try { + pValues = await fetchSignificantTermPValues( + esClient, + params, + [fieldCandidate], + undefined, + sampleProbability, + (e) => { + console.log('fetchSignificantTermPValues ERROR', e); + }, + abortSignal + ); + } catch (e) { + console.log('catch fetchSignificantTermPValues ERROR', e); + return; + } + + if (pValues.length > 0) { + pValues.forEach((d) => { + fieldsToSample.add(d.fieldName); + }); + significantTerms.push(...pValues); + } + } else if (isTextFieldCandidate(payload)) { + const { textFieldCandidate } = payload; + + const significantCategoriesForField = await fetchSignificantCategories( + esClient, + params, + [textFieldCandidate], + undefined, + sampleProbability, + (e) => { + console.log('fetchSignificantCategories ERROR', e); + }, + abortSignal + ); + + if (significantCategoriesForField.length > 0) { + significantCategories.push(...significantCategoriesForField); + } + } + }, MAX_CONCURRENT_QUERIES); + + pValuesQueue.push( + [ + ...textFieldCandidates.map((d) => ({ textFieldCandidate: d })), + ...fieldCandidates.map((d) => ({ fieldCandidate: d })), + ], + (err) => { + if (err) { + console.error('queue push error', err); + pValuesQueue.kill(); + } + } + ); + await pValuesQueue.drain(); + + fieldValuePairsCount = significantCategories.length + significantTerms.length; + + const debugEndTime = Date.now(); + const debugDelta = (debugEndTime - debugStartTime) / 1000; + console.log(`Took: ${debugDelta}s`); + + // RETURN DATA + + const logRateChange: LogRateChange = { + type: logRateType, + timestamp: changePoint.key, + logRateChangeCount: changePointDocCount, + averageLogRateCount: Math.round(mean(Object.values(dateHistogramBuckets)) ?? 0), + logRateAggregationIntervalUsedForAnalysis: moment + .duration(Math.round(intervalMs / 1000), 'seconds') + .humanize(), + ...(sampleProbability < 1 ? { documentSamplingFactorForAnalysis: sampleProbability } : {}), + extendedChangePoint: { + startTs: changePoint.startTs, + endTs: changePoint.endTs, + }, + }; + + if (fieldValuePairsCount === 0) { + return { logRateChange, significantItems: [], dateHistogramBuckets, windowParameters }; + } + + const significantItems: SimpleSignificantItem[] = [...significantTerms, ...significantCategories] + .filter(({ bg_count: bgCount, doc_count: docCount }) => { + return docCount > bgCount; + }) + .map(({ fieldName, fieldValue, type, doc_count: docCount, bg_count: bgCount }) => ({ + field: fieldName, + value: fieldValue, + type: (type === 'keyword' + ? 'metadata' + : 'log message pattern') as SimpleSignificantItem['type'], + documentCount: docCount, + baselineCount: bgCount, + logRateChangeSort: bgCount > 0 ? docCount / bgCount : docCount, + logRateChange: + bgCount > 0 + ? logRateType === 'spike' + ? `${Math.round((docCount / bgCount) * 100) / 100}x increase` + : `${Math.round((bgCount / docCount) * 100) / 100}x decrease` + : logRateType === 'spike' + ? `${docCount} docs up from 0 in baseline` + : `0 docs down from ${docCount} in baseline`, + })) + .sort((a, b) => b.logRateChangeSort - a.logRateChangeSort); + + return { logRateChange, significantItems, dateHistogramBuckets, windowParameters }; +}; diff --git a/x-pack/plugins/aiops_api/common/types.ts b/x-pack/plugins/aiops_api/common/types.ts index 07eb3a34ece67f..97d29635d1038e 100644 --- a/x-pack/plugins/aiops_api/common/types.ts +++ b/x-pack/plugins/aiops_api/common/types.ts @@ -5,39 +5,30 @@ * 2.0. */ +import type { + LogRateChange, + SimpleSignificantItem, +} from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; + export interface GetAiopsLogRateAnalysisFunctionResponse { content: | string | { - logRateChange: { - type: string; - spikeLogRate?: number; - dipLogRate?: number; - averageLogRate: number; - logRateAggregationIntervalUsedForAnalysis: string; - documentSamplingFactorForAnalysis?: number; - }; - significantItems: Array<{ - field: string; - value: string | number; - type: 'metadata' | 'log message pattern'; - documentCount: number; - baselineCount: number; - logIncrease: string; - }>; + logRateChange: Pick< + LogRateChange, + | 'type' + | 'logRateChangeCount' + | 'averageLogRateCount' + | 'logRateAggregationIntervalUsedForAnalysis' + | 'documentSamplingFactorForAnalysis' + >; + significantItems: SimpleSignificantItem[]; }; + data: { dateHistogram: Record; logRateAnalysisUILink: string; - logRateChange: { - type: string; - timestamp: string; - spikeLogRate?: number; - dipLogRate?: number; - averageLogRate: number; - logRateAggregationIntervalUsedForAnalysis: string; - documentSamplingFactorForAnalysis?: number; - extendedChangePoint: { startTs: number; endTs: number }; - }; + logRateChange: LogRateChange; + significantItems: SimpleSignificantItem[]; }; } diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index d936d32a6023d0..0b6829b52b7e12 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -5,37 +5,18 @@ * 2.0. */ -import { queue } from 'async'; import { FromSchema } from 'json-schema-to-ts'; -import { mean } from 'd3-array'; -import moment from 'moment'; import dedent from 'dedent'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - import rison from '@kbn/rison'; -import { type SignificantItem } from '@kbn/ml-agg-utils'; import { i18n } from '@kbn/i18n'; -import dateMath from '@kbn/datemath'; -import { - getExtendedChangePoint, - getWindowParametersForTrigger, -} from '@kbn/aiops-log-rate-analysis'; -import { fetchIndexInfo } from '@kbn/aiops-log-rate-analysis/queries/fetch_index_info'; -import type { AiopsLogRateAnalysisSchema } from '@kbn/aiops-log-rate-analysis/api/schema'; -import { fetchSignificantCategories } from '@kbn/aiops-log-rate-analysis/queries/fetch_significant_categories'; -import { fetchSignificantTermPValues } from '@kbn/aiops-log-rate-analysis/queries/fetch_significant_term_p_values'; -import { getSampleProbability } from '@kbn/ml-random-sampler-utils'; +import { fetchSimpleLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; import type { GetAiopsLogRateAnalysisFunctionResponse } from '../../common/types'; import { FunctionRegistrationParameters } from '.'; // import { ApmTimeseries, getApmTimeseries } from '../routes/assistant_functions/get_apm_timeseries'; -// Don't use more than 10 here otherwise Kibana will emit an error -// regarding a limit of abort signal listeners of more than 10. -export const MAX_CONCURRENT_QUERIES = 5; - export const NON_EMPTY_STRING = { type: 'string' as const, minLength: 1, @@ -96,295 +77,35 @@ export function registerGetAiopsLogRateAnalysisFunction({ parameters, }, async ({ arguments: args }, abortSignal): Promise => { - const debugStartTime = Date.now(); const { esClient } = resources; - // CHANGE POINT DETECTION - - const barTarget = 75; - const earliestMs = dateMath.parse(args.start)?.valueOf(); - const latestMs = dateMath.parse(args.end, { roundUp: true })?.valueOf(); - - if (earliestMs === undefined || latestMs === undefined) { - return { content: 'Could not parse time range.', data: {} }; - } - const delta = latestMs - earliestMs; - - const dayMs = 86400 * 1000; - const dayThreshold = dayMs * 22; - - const weekMs = dayMs * 7; - const weekThreshold = weekMs * 22; - - const monthMs = dayMs * 30; - const monthThreshold = monthMs * 22; - - let intervalMs = Math.round(delta / barTarget); - - if (delta > monthThreshold) { - intervalMs = monthMs; - } else if (delta > weekThreshold) { - intervalMs = weekMs; - } else if (delta > dayThreshold) { - intervalMs = dayMs; - } - - const aggs: Record = { - eventRate: { - date_histogram: { - field: args.timefield, - fixed_interval: `${intervalMs}ms`, - min_doc_count: 0, - ...(earliestMs !== undefined && latestMs !== undefined - ? { - extended_bounds: { - min: earliestMs, - max: latestMs, - }, - } - : {}), - }, - }, - change_point_request: { - // @ts-expect-error missing from ES spec - change_point: { - buckets_path: 'eventRate>_count', - }, - }, - }; - - const searchQuery = { - range: { - [args.timefield]: { - gte: earliestMs, - lte: latestMs, - format: 'epoch_millis', - }, - }, - }; - const searchBody: estypes.MsearchMultisearchBody = { - query: searchQuery, - aggs, - track_total_hits: false, - size: 0, - }; - - const histogram = await esClient.search( - { - index: args.index, - body: searchBody, - }, - { - // signal: abortSignal, - maxRetries: 0, - } - ); - - if (histogram.aggregations.change_point_request.bucket === undefined) { - return { content: 'No log rate change detected.', data: [] }; - } - - const buckets = histogram.aggregations.eventRate.buckets.reduce((acc, cur) => { - acc[cur.key] = cur.doc_count; - return acc; - }, {}); - - const changePointTs = dateMath - .parse(histogram.aggregations.change_point_request.bucket.key) - ?.valueOf(); - - if (changePointTs === undefined) { - return { content: 'There was an error parsing the log rate change timestamp.', data: [] }; - } - - const extendedChangePoint = getExtendedChangePoint(buckets, changePointTs); - let logRateType = Object.keys(histogram.aggregations.change_point_request.type)[0]; - - // FIELD CANDIDATES - - const fieldCandidates: string[] = []; - const textFieldCandidates: string[] = []; - const changePoint = { - ...extendedChangePoint, - key: changePointTs, - type: logRateType, - }; - const wp = getWindowParametersForTrigger( - extendedChangePoint.startTs, - intervalMs, - earliestMs, - latestMs, - changePoint - ); - - const indexInfoParams: AiopsLogRateAnalysisSchema = { - index: args.index, - start: earliestMs, - end: latestMs, - searchQuery: JSON.stringify(searchQuery), - timeFieldName: args.timefield, - ...wp, - }; - - const indexInfo = await fetchIndexInfo( + const resp = await fetchSimpleLogRateAnalysis( esClient, - indexInfoParams, - ['message', 'error.message'], + args.index, + args.start, + args.end, + args.timefield, abortSignal ); - const baselineNumBuckets = (wp.baselineMax - wp.baselineMin) / intervalMs; - const baselinePerBucket = indexInfo.baselineTotalDocCount / baselineNumBuckets; - - const deviationNumBuckets = (wp.deviationMax - wp.deviationMin) / intervalMs; - const deviationPerBucket = indexInfo.deviationTotalDocCount / deviationNumBuckets; - - const analysisWindowParameters = - deviationPerBucket > baselinePerBucket - ? wp - : { - baselineMin: wp.deviationMin, - baselineMax: wp.deviationMax, - deviationMin: wp.baselineMin, - deviationMax: wp.baselineMax, - }; - - logRateType = deviationPerBucket > baselinePerBucket ? 'spike' : 'dip'; - const logRateAttributeName = `${logRateType}LogRate`; - - const params: AiopsLogRateAnalysisSchema = { - ...indexInfoParams, - ...analysisWindowParameters, - }; - - fieldCandidates.push(...indexInfo.fieldCandidates); - textFieldCandidates.push(...indexInfo.textFieldCandidates); - const sampleProbability = getSampleProbability( - indexInfo.deviationTotalDocCount + indexInfo.baselineTotalDocCount - ); - - // SIGNIFICANT ITEMS - - // This will store the combined count of detected significant log patterns and keywords - let fieldValuePairsCount = 0; - - const significantCategories: SignificantItem[] = []; - - // Get significant categories of text fields - if (textFieldCandidates.length > 0) { - significantCategories.push( - ...(await fetchSignificantCategories( - esClient, - params, - textFieldCandidates, - undefined, - sampleProbability, - () => {}, - abortSignal - )) - ); - } - - const significantTerms: SignificantItem[] = []; - const fieldsToSample = new Set(); - - const pValuesQueue = queue(async function (fieldCandidate: string) { - const pValues = await fetchSignificantTermPValues( - esClient, - params, - [fieldCandidate], - undefined, - sampleProbability, - () => {}, - abortSignal - ); - - if (pValues.length > 0) { - pValues.forEach((d) => { - fieldsToSample.add(d.fieldName); - }); - significantTerms.push(...pValues); - } - }, MAX_CONCURRENT_QUERIES); - - pValuesQueue.push(fieldCandidates, (err) => { - if (err) { - pValuesQueue.kill(); - } - }); - await pValuesQueue.drain(); - - fieldValuePairsCount = significantCategories.length + significantTerms.length; - - if (fieldValuePairsCount === 0) { - return { content: 'Log rate analysis did not identify any significant items.', data: [] }; - } - - // RETURN DATA - const debugEndTime = Date.now(); - const debugDelta = (debugEndTime - debugStartTime) / 1000; - console.log(`Took: ${debugDelta}s`); - - const logRateChange = { - type: Object.keys(histogram.aggregations.change_point_request.type)[0], - timestamp: histogram.aggregations.change_point_request.bucket.key, - [logRateAttributeName]: histogram.aggregations.change_point_request.bucket.doc_count, - averageLogRate: Math.round(mean(Object.values(buckets)) ?? 0), - logRateAggregationIntervalUsedForAnalysis: moment - .duration(Math.round(intervalMs / 1000), 'seconds') - .humanize(), - ...(sampleProbability < 1 ? { documentSamplingFactorForAnalysis: sampleProbability } : {}), - extendedChangePoint, - }; - - const significantItems = [...significantTerms, ...significantCategories] - .filter(({ bg_count, doc_count }) => { - return doc_count > bg_count; - }) - .map(({ fieldName, fieldValue, type, doc_count, bg_count }) => ({ - field: fieldName, - value: fieldValue, - type: type === 'keyword' ? 'metadata' : 'log message pattern', - documentCount: doc_count, - baselineCount: bg_count, - logRateChangeSort: bg_count > 0 ? doc_count / bg_count : doc_count, - logRateChange: - bg_count > 0 - ? logRateType === 'spike' - ? `${Math.round((doc_count / bg_count) * 100) / 100}x increase` - : `${Math.round((bg_count / doc_count) * 100) / 100}x decrease` - : logRateType === 'spike' - ? `${doc_count} docs up from 0 in baseline` - : `0 docs down from ${doc_count} in baseline`, - })) - .sort((a, b) => b.logRateChangeSort - a.logRateChangeSort); + const { logRateChange, significantItems, dateHistogramBuckets, windowParameters } = resp; return { content: { - logRateChange: { - type: Object.keys(histogram.aggregations.change_point_request.type)[0], - [logRateAttributeName]: histogram.aggregations.change_point_request.bucket.doc_count, - averageLogRate: Math.round(mean(Object.values(buckets)) ?? 0), - logRateAggregationIntervalUsedForAnalysis: moment - .duration(Math.round(intervalMs / 1000), 'seconds') - .humanize(), - ...(sampleProbability < 1 - ? { documentSamplingFactorForAnalysis: sampleProbability } - : {}), - }, + logRateChange, significantItems: significantItems.slice(0, 100), }, data: { - dateHistogram: buckets, + dateHistogram: dateHistogramBuckets, logRateAnalysisUILink: `/app/ml/aiops/log_rate_analysis?index=${ args.dataViewId }&_a=${rison.encode({ logRateAnalysis: { wp: { - bMin: wp.baselineMin, - bMax: wp.baselineMax, - dMin: wp.deviationMin, - dMax: wp.deviationMax, + bMin: windowParameters.baselineMin, + bMax: windowParameters.baselineMax, + dMin: windowParameters.deviationMin, + dMax: windowParameters.deviationMax, }, }, })}`, From b60ea314ee871bb17e92ea9b62db053a9ab2d45f Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Wed, 22 May 2024 16:53:20 +0200 Subject: [PATCH 21/31] make logger/emitError optional --- .../queries/fetch_simple_log_rate_analysis.ts | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts index b0d9fe41a37052..2a1443af4d6358 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts @@ -26,19 +26,19 @@ import { fetchSignificantTermPValues } from './fetch_significant_term_p_values'; // regarding a limit of abort signal listeners of more than 10. const MAX_CONCURRENT_QUERIES = 5; -interface FieldCandidate { - fieldCandidate: string; +interface KeywordFieldCandidate { + keywordFieldCandidate: string; } -const isFieldCandidate = (d: unknown): d is FieldCandidate => - isPopulatedObject(d, ['fieldCandidate']); +const isKeywordFieldCandidate = (d: unknown): d is KeywordFieldCandidate => + isPopulatedObject(d, ['keywordFieldCandidate']); interface TextFieldCandidate { textFieldCandidate: string; } -const isTextFieldCandidate = (d: unknown): d is FieldCandidate => +const isTextFieldCandidate = (d: unknown): d is TextFieldCandidate => isPopulatedObject(d, ['textFieldCandidate']); -type Candidate = FieldCandidate | TextFieldCandidate; +type QueueFieldCandidate = KeywordFieldCandidate | TextFieldCandidate; export interface LogRateChange { type: string; @@ -118,7 +118,7 @@ export const fetchSimpleLogRateAnalysis = async ( // FIELD CANDIDATES - const fieldCandidates: string[] = []; + const keywordFieldCandidates: string[] = []; const textFieldCandidates: string[] = []; const indexInfoParams: AiopsLogRateAnalysisSchema = { @@ -162,7 +162,7 @@ export const fetchSimpleLogRateAnalysis = async ( ...analysisWindowParameters, }; - fieldCandidates.push(...indexInfo.fieldCandidates); + keywordFieldCandidates.push(...indexInfo.fieldCandidates); textFieldCandidates.push(...indexInfo.textFieldCandidates); const sampleProbability = getSampleProbability( indexInfo.deviationTotalDocCount + indexInfo.baselineTotalDocCount @@ -177,27 +177,20 @@ export const fetchSimpleLogRateAnalysis = async ( const significantTerms: SignificantItem[] = []; const fieldsToSample = new Set(); - const pValuesQueue = queue(async function (payload: Candidate) { - if (isFieldCandidate(payload)) { - const { fieldCandidate } = payload; + const pValuesQueue = queue(async function (payload: QueueFieldCandidate) { + if (isKeywordFieldCandidate(payload)) { + const { keywordFieldCandidate } = payload; let pValues: Awaited> = []; - try { - pValues = await fetchSignificantTermPValues( - esClient, - params, - [fieldCandidate], - undefined, - sampleProbability, - (e) => { - console.log('fetchSignificantTermPValues ERROR', e); - }, - abortSignal - ); - } catch (e) { - console.log('catch fetchSignificantTermPValues ERROR', e); - return; - } + pValues = await fetchSignificantTermPValues( + esClient, + params, + [keywordFieldCandidate], + undefined, + sampleProbability, + undefined, + abortSignal + ); if (pValues.length > 0) { pValues.forEach((d) => { @@ -214,9 +207,7 @@ export const fetchSimpleLogRateAnalysis = async ( [textFieldCandidate], undefined, sampleProbability, - (e) => { - console.log('fetchSignificantCategories ERROR', e); - }, + undefined, abortSignal ); @@ -229,11 +220,10 @@ export const fetchSimpleLogRateAnalysis = async ( pValuesQueue.push( [ ...textFieldCandidates.map((d) => ({ textFieldCandidate: d })), - ...fieldCandidates.map((d) => ({ fieldCandidate: d })), + ...keywordFieldCandidates.map((d) => ({ keywordFieldCandidate: d })), ], (err) => { if (err) { - console.error('queue push error', err); pValuesQueue.kill(); } } From 163be0a4021daa82264a379542e2881711eb0e8c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 May 2024 10:00:40 +0200 Subject: [PATCH 22/31] update API endpoint --- .../routes/log_rate_analysis/define_route.ts | 9 ++- .../route_handler_factory.ts | 59 ++++++++++++++----- .../server/routes/log_rate_analysis/schema.ts | 18 ++++++ 3 files changed, 69 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts index d121ed70ac9f0a..b4bbeeeb62b651 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/define_route.ts @@ -14,6 +14,7 @@ import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; import type { AiopsApiLicense } from '../../types'; +import { aiopsLogRateAnalysisSchema } from './schema'; import { routeHandlerFactory } from './route_handler_factory'; /** @@ -31,8 +32,12 @@ export const defineRoute = ( ) => { router.post( { - path: '/api/aiops/log_rate_analysis', - validate: false, + path: '/internal/aiops/simple_log_rate_analysis', + validate: { + request: { + body: aiopsLogRateAnalysisSchema, + }, + }, }, routeHandlerFactory('1', license, logger, coreStart, usageCounter) ); diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts index 2b2d1fcf6d109d..64314c662e9747 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -13,35 +13,37 @@ import type { KibanaResponseFactory, } from '@kbn/core/server'; import type { Logger } from '@kbn/logging'; -// import { createExecutionContext } from '@kbn/ml-route-utils'; +import { createExecutionContext } from '@kbn/ml-route-utils'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; -// import { AIOPS_TELEMETRY_ID, AIOPS_PLUGIN_ID } from '@kbn/aiops-common/constants'; -import type { - AiopsLogRateAnalysisSchema, - AiopsLogRateAnalysisApiVersion as ApiVersion, -} from '@kbn/aiops-log-rate-analysis/api/schema'; +import { + // AIOPS_TELEMETRY_ID, + AIOPS_API_PLUGIN_ID, +} from '@kbn/aiops-common/constants'; // import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants'; // import { isRequestAbortedError } from '@kbn/aiops-common/is_request_aborted_error'; +import { fetchSimpleLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; // import { trackAIOpsRouteUsage } from '../../lib/track_route_usage'; import type { AiopsApiLicense } from '../../types'; +import type { AiopsLogRateAnalysisSchema } from './schema'; + /** * Log rate analysis route handler. */ -export function routeHandlerFactory( - version: T, +export function routeHandlerFactory( + version: '1', license: AiopsApiLicense, logger: Logger, coreStart: CoreStart, usageCounter?: UsageCounter -): RequestHandler> { +): RequestHandler { return async ( context: RequestHandlerContext, - request: KibanaRequest>, + request: KibanaRequest, response: KibanaResponseFactory ) => { - // const { headers } = request; + const { events } = request; // trackAIOpsRouteUsage( // `POST ${AIOPS_API_ENDPOINT.LOG_RATE_ANALYSIS}`, @@ -53,11 +55,38 @@ export function routeHandlerFactory( return response.forbidden(); } - // const client = (await context.core).elasticsearch.client.asCurrentUser; - // const executionContext = createExecutionContext(coreStart, AIOPS_PLUGIN_ID, request.route.path); + const client = (await context.core).elasticsearch.client.asCurrentUser; + const executionContext = createExecutionContext( + coreStart, + AIOPS_API_PLUGIN_ID, + request.route.path + ); + + const controller = new AbortController(); + const abortSignal = controller.signal; + + events.aborted$.subscribe(() => { + controller.abort(); + }); + events.completed$.subscribe(() => { + controller.abort(); + }); + + const { index, timefield, start, end } = request.body; + + return await coreStart.executionContext.withContext(executionContext, async () => { + const logRateAnalysis = await fetchSimpleLogRateAnalysis( + client, + index, + start, + end, + timefield, + abortSignal + ); - return response.ok({ - body: 'CAN HAS LOG RATE ANALYSIS', + return response.ok({ + body: logRateAnalysis, + }); }); }; } diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts new file mode 100644 index 00000000000000..8e52bc165e97b5 --- /dev/null +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { TypeOf } from '@kbn/config-schema'; +import { schema } from '@kbn/config-schema'; + +export const aiopsLogRateAnalysisSchema = schema.object({ + index: schema.string(), + start: schema.string(), + end: schema.string(), + timefield: schema.string(), +}); + +export type AiopsLogRateAnalysisSchema = TypeOf; From aa48700d10915f8bf7fee31df542ff68ac8c6b21 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 24 May 2024 11:08:41 +0200 Subject: [PATCH 23/31] API: field candidates support --- .../queries/fetch_change_point_detection.ts | 2 +- .../queries/fetch_index_info.ts | 103 ++++++++++-------- .../queries/fetch_simple_log_rate_analysis.ts | 11 +- .../route_handler_factory.ts | 7 +- .../server/routes/log_rate_analysis/schema.ts | 2 + 5 files changed, 70 insertions(+), 55 deletions(-) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts index bd8cf603de556d..b61ca205ef6a5c 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_change_point_detection.ts @@ -47,7 +47,7 @@ export const fetchChangePointDetection = async ( latestMs: number, timefield: string, searchQuery: estypes.QueryDslQueryContainer, - abortSignal: AbortSignal + abortSignal?: AbortSignal ): Promise => { const barTarget = 75; diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts index a97b6049ab7b54..7102329dc2a1bc 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts @@ -52,48 +52,70 @@ export const fetchIndexInfo = async ({ }): Promise => { const { textFieldCandidatesOverrides = [], ...params } = args; const { index } = params; - // Get all supported fields - const respMapping = await esClient.fieldCaps( - { - fields: '*', - filters: '-metadata', - include_empty_fields: false, - index, - index_filter: { - range: { - [params.timeFieldName]: { - gte: params.deviationMin, - lte: params.deviationMax, - }, - }, - }, - types: [...SUPPORTED_ES_FIELD_TYPES, ...SUPPORTED_ES_FIELD_TYPES_TEXT], - }, - { signal: abortSignal, maxRetries: 0 } - ); const allFieldNames: string[] = []; const acceptableFields: Set = new Set(); const acceptableTextFields: Set = new Set(); - Object.entries(respMapping.fields).forEach(([key, value]) => { - const fieldTypes = Object.keys(value) as ES_FIELD_TYPES[]; - const isSupportedType = fieldTypes.some((type) => SUPPORTED_ES_FIELD_TYPES.includes(type)); - const isAggregatable = fieldTypes.some((type) => value[type].aggregatable); - const isTextField = fieldTypes.some((type) => SUPPORTED_ES_FIELD_TYPES_TEXT.includes(type)); + let fieldCandidates: string[] = []; + let textFieldCandidates: string[] = []; + + if (includeFieldCandidates) { + // Get all supported fields + const respMapping = await esClient.fieldCaps( + { + fields: '*', + filters: '-metadata,-parent', + include_empty_fields: false, + index, + index_filter: { + range: { + [params.timeFieldName]: { + gte: params.deviationMin, + lte: params.deviationMax, + }, + }, + }, + types: [...SUPPORTED_ES_FIELD_TYPES, ...SUPPORTED_ES_FIELD_TYPES_TEXT], + }, + { signal: abortSignal, maxRetries: 0 } + ); - // Check if fieldName is something we can aggregate on - if (isSupportedType && isAggregatable) { - acceptableFields.add(key); - } + Object.entries(respMapping.fields).forEach(([key, value]) => { + const fieldTypes = Object.keys(value) as ES_FIELD_TYPES[]; + const isSupportedType = fieldTypes.some((type) => SUPPORTED_ES_FIELD_TYPES.includes(type)); + const isAggregatable = fieldTypes.some((type) => value[type].aggregatable); + const isTextField = fieldTypes.some((type) => SUPPORTED_ES_FIELD_TYPES_TEXT.includes(type)); - if (isTextField && TEXT_FIELD_WHITE_LIST.includes(key)) { - acceptableTextFields.add(key); - } + // Check if fieldName is something we can aggregate on + if (isSupportedType && isAggregatable) { + acceptableFields.add(key); + } - allFieldNames.push(key); - }); + if (isTextField && TEXT_FIELD_WHITE_LIST.includes(key)) { + acceptableTextFields.add(key); + } + + allFieldNames.push(key); + }); + + const textFieldCandidatesOverridesWithKeywordPostfix = textFieldCandidatesOverrides.map( + (d) => `${d}.keyword` + ); + + fieldCandidates = [...acceptableFields].filter( + (field) => !textFieldCandidatesOverridesWithKeywordPostfix.includes(field) + ); + textFieldCandidates = [...acceptableTextFields].filter((field) => { + const fieldName = field.replace(new RegExp(/\.text$/), ''); + return ( + (!fieldCandidates.includes(fieldName) && + !fieldCandidates.includes(`${fieldName}.keyword`)) || + textFieldCandidatesOverrides.includes(field) + ); + }); + } // Get the total doc count for the baseline time range const respBaselineTotalDocCount = await esClient.search( @@ -113,21 +135,6 @@ export const fetchIndexInfo = async ({ } ); - const textFieldCandidatesOverridesWithKeywordPostfix = textFieldCandidatesOverrides.map( - (d) => `${d}.keyword` - ); - - const fieldCandidates: string[] = [...acceptableFields].filter( - (field) => !textFieldCandidatesOverridesWithKeywordPostfix.includes(field) - ); - const textFieldCandidates: string[] = [...acceptableTextFields].filter((field) => { - const fieldName = field.replace(new RegExp(/\.text$/), ''); - return ( - (!fieldCandidates.includes(fieldName) && !fieldCandidates.includes(`${fieldName}.keyword`)) || - textFieldCandidatesOverrides.includes(field) - ); - }); - const baselineTotalDocCount = (respBaselineTotalDocCount.hits.total as estypes.SearchTotalHits) .value; const deviationTotalDocCount = (respDeviationTotalDocCount.hits.total as estypes.SearchTotalHits) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts index 2a1443af4d6358..27f89b21373fc4 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts @@ -77,7 +77,9 @@ export const fetchSimpleLogRateAnalysis = async ( start: string, end: string, timefield: string, - abortSignal: AbortSignal + abortSignal?: AbortSignal, + keywordFieldCandidates: string[] = [], + textFieldCandidates: string[] = [] ) => { const debugStartTime = Date.now(); @@ -118,8 +120,8 @@ export const fetchSimpleLogRateAnalysis = async ( // FIELD CANDIDATES - const keywordFieldCandidates: string[] = []; - const textFieldCandidates: string[] = []; + const includeFieldCandidates = + keywordFieldCandidates.length === 0 && textFieldCandidates.length === 0; const indexInfoParams: AiopsLogRateAnalysisSchema = { index, @@ -134,7 +136,8 @@ export const fetchSimpleLogRateAnalysis = async ( esClient, indexInfoParams, ['message', 'error.message'], - abortSignal + abortSignal, + includeFieldCandidates ); const baselineNumBuckets = diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts index 64314c662e9747..4620580b847482 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -72,7 +72,8 @@ export function routeHandlerFactory( controller.abort(); }); - const { index, timefield, start, end } = request.body; + const { index, timefield, start, end, keywordFieldCandidates, textFieldCandidates } = + request.body; return await coreStart.executionContext.withContext(executionContext, async () => { const logRateAnalysis = await fetchSimpleLogRateAnalysis( @@ -81,7 +82,9 @@ export function routeHandlerFactory( start, end, timefield, - abortSignal + abortSignal, + keywordFieldCandidates, + textFieldCandidates ); return response.ok({ diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts index 8e52bc165e97b5..51d10fe6aee9e2 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/schema.ts @@ -13,6 +13,8 @@ export const aiopsLogRateAnalysisSchema = schema.object({ start: schema.string(), end: schema.string(), timefield: schema.string(), + keywordFieldCandidates: schema.arrayOf(schema.string(), { defaultValue: [] }), + textFieldCandidates: schema.arrayOf(schema.string(), { defaultValue: [] }), }); export type AiopsLogRateAnalysisSchema = TypeOf; From aa013fdfe8b95a4c6937caafcdb350261fd0f207 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Tue, 11 Jun 2024 11:06:51 +0200 Subject: [PATCH 24/31] comments --- .../queries/fetch_simple_log_rate_analysis.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts index 27f89b21373fc4..cd1ca9086f7200 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts @@ -69,6 +69,8 @@ export interface SimpleSignificantItem { * @param end The end of the time range, in Elasticsearch date math, like `now-24h`. * @param timefield The Elasticesarch source index pattern time field. * @param abortSignal Abort signal. + * @param keywordFieldCandidates Optional keyword field candidates. + * @param textFieldCandidates Optional text field candidates. * @returns Log rate analysis data. */ export const fetchSimpleLogRateAnalysis = async ( From 4aa57e8b54516b133b61e2e2ab356b93af061108 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 17 Jun 2024 11:11:52 +0200 Subject: [PATCH 25/31] log rate analysis for alert details page --- .../get_apm_service_summary/index.ts | 1 + .../index.ts | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_apm_service_summary/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_apm_service_summary/index.ts index d28152127648b3..2d7c986bb66782 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_apm_service_summary/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_apm_service_summary/index.ts @@ -82,6 +82,7 @@ export async function getApmServiceSummary({ apmAlertsClient: ApmAlertsClient; logger: Logger; }): Promise { + console.log('SERVICE SUMMARY!!'); const start = datemath.parse(args.start)?.valueOf()!; const end = datemath.parse(args.end)?.valueOf()!; diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts index 4e9ae1b546aa7d..f34936c0a1c592 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts @@ -12,6 +12,8 @@ import type { } from '@kbn/observability-plugin/server/services'; import moment from 'moment'; import { isEmpty } from 'lodash'; +import { fetchSimpleLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; +import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server'; import { SERVICE_NAME, SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../../common/es_fields/apm'; import { getApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client'; import { getApmEventClient } from '../../../lib/helpers/get_apm_event_client'; @@ -160,6 +162,24 @@ export const getAlertDetailsContextHandler = ( }); } + // log rate analysis + dataFetchers.push(async () => { + const index = await coreContext.uiSettings.client.get(aiAssistantLogsIndexPattern); + const logRateAnalysis = await fetchSimpleLogRateAnalysis( + esClient, + index, + moment(alertStartedAt).subtract(15, 'minute').toISOString(), + moment(alertStartedAt).add(15, 'minute').toISOString(), + '@timestamp' + ); + + return { + key: 'logRateAnalysis', + description: `Significant field/value pairs in log data that contributed to changes in the log rate.`, + data: logRateAnalysis, + }; + }); + // log categories dataFetchers.push(async () => { const downstreamDependencies = await downstreamDependenciesPromise; From 59bf1e795b731efd400e7de0a81c891926dd3334 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 17 Jun 2024 16:47:42 +0200 Subject: [PATCH 26/31] localhost/ssl support for scripts/synthtrace --- .../src/cli/utils/get_apm_es_client.ts | 2 ++ .../src/cli/utils/get_assets_es_client.ts | 2 ++ .../src/cli/utils/get_infra_es_client.ts | 2 ++ .../src/cli/utils/get_logs_es_client.ts | 2 ++ .../src/cli/utils/get_service_urls.ts | 7 ++++- .../kbn-apm-synthtrace/src/cli/utils/ssl.ts | 31 +++++++++++++++++++ .../client/apm_synthtrace_kibana_client.ts | 4 +++ .../infra/infra_synthtrace_kibana_client.ts | 3 ++ 8 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 packages/kbn-apm-synthtrace/src/cli/utils/ssl.ts diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts index ee350ab3c56d8e..b876f89fa38e0b 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_apm_es_client.ts @@ -10,6 +10,7 @@ import { Client } from '@elastic/elasticsearch'; import { ApmSynthtraceEsClient } from '../../..'; import { Logger } from '../../lib/utils/create_logger'; import { RunOptions } from './parse_run_cli_flags'; +import { getEsClientTlsSettings } from './ssl'; export function getApmEsClient({ target, @@ -23,6 +24,7 @@ export function getApmEsClient({ }) { const client = new Client({ node: target, + tls: getEsClientTlsSettings(target), }); const apmEsClient = new ApmSynthtraceEsClient({ diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_assets_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_assets_es_client.ts index 059a837492699a..fa44bcbe3b2dea 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/get_assets_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_assets_es_client.ts @@ -10,6 +10,7 @@ import { Client } from '@elastic/elasticsearch'; import { AssetsSynthtraceEsClient } from '../../lib/assets/assets_synthtrace_es_client'; import { Logger } from '../../lib/utils/create_logger'; import { RunOptions } from './parse_run_cli_flags'; +import { getEsClientTlsSettings } from './ssl'; export function getAssetsEsClient({ target, @@ -21,6 +22,7 @@ export function getAssetsEsClient({ }) { const client = new Client({ node: target, + tls: getEsClientTlsSettings(target), }); return new AssetsSynthtraceEsClient({ diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts index 82a714dc8f244c..a0f651d8f3185a 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_infra_es_client.ts @@ -10,6 +10,7 @@ import { Client } from '@elastic/elasticsearch'; import { InfraSynthtraceEsClient } from '../../lib/infra/infra_synthtrace_es_client'; import { Logger } from '../../lib/utils/create_logger'; import { RunOptions } from './parse_run_cli_flags'; +import { getEsClientTlsSettings } from './ssl'; export function getInfraEsClient({ target, @@ -21,6 +22,7 @@ export function getInfraEsClient({ }) { const client = new Client({ node: target, + tls: getEsClientTlsSettings(target), }); return new InfraSynthtraceEsClient({ diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts index 8bfcdaef908399..863cf2c9964d41 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_logs_es_client.ts @@ -10,6 +10,7 @@ import { Client } from '@elastic/elasticsearch'; import { LogsSynthtraceEsClient } from '../../lib/logs/logs_synthtrace_es_client'; import { Logger } from '../../lib/utils/create_logger'; import { RunOptions } from './parse_run_cli_flags'; +import { getEsClientTlsSettings } from './ssl'; export function getLogsEsClient({ target, @@ -21,6 +22,7 @@ export function getLogsEsClient({ }) { const client = new Client({ node: target, + tls: getEsClientTlsSettings(target), }); return new LogsSynthtraceEsClient({ diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts index 3967040a569af9..d8c11af6b41a98 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_service_urls.ts @@ -10,6 +10,7 @@ import fetch from 'node-fetch'; import { format, parse, Url } from 'url'; import { Logger } from '../../lib/utils/create_logger'; import { RunOptions } from './parse_run_cli_flags'; +import { getFetchAgent } from './ssl'; async function discoverAuth(parsedTarget: Url) { const possibleCredentials = [`admin:changeme`, `elastic:changeme`, `elastic_serverless:changeme`]; @@ -20,7 +21,9 @@ async function discoverAuth(parsedTarget: Url) { }); let status: number; try { - const response = await fetch(url); + const response = await fetch(url, { + agent: getFetchAgent(url), + }); status = response.status; } catch (err) { status = 0; @@ -43,6 +46,7 @@ async function getKibanaUrl({ target, logger }: { target: string; logger: Logger method: 'HEAD', follow: 1, redirect: 'manual', + agent: getFetchAgent(target), }); const discoveredKibanaUrl = @@ -62,6 +66,7 @@ async function getKibanaUrl({ target, logger }: { target: string; logger: Logger const redirectedResponse = await fetch(discoveredKibanaUrlWithAuth, { method: 'HEAD', + agent: getFetchAgent(discoveredKibanaUrlWithAuth), }); if (redirectedResponse.status !== 200) { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/ssl.ts b/packages/kbn-apm-synthtrace/src/cli/utils/ssl.ts new file mode 100644 index 00000000000000..22ad438323e6b1 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/cli/utils/ssl.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as Fs from 'fs'; +import { CA_CERT_PATH } from '@kbn/dev-utils'; +import https from 'https'; + +export function getFetchAgent(url: string) { + const isHTTPS = new URL(url).protocol === 'https:'; + const isLocalhost = new URL(url).hostname === 'localhost'; + return isHTTPS && isLocalhost ? new https.Agent({ rejectUnauthorized: false }) : undefined; +} + +export function getEsClientTlsSettings(url: string) { + const isHTTPS = new URL(url).protocol === 'https:'; + // load the CA cert from disk if necessary + const caCert = isHTTPS ? Fs.readFileSync(CA_CERT_PATH) : null; + const isLocalhost = new URL(url).hostname === 'localhost'; + + return caCert && isLocalhost + ? { + ca: caCert, + rejectUnauthorized: true, + } + : undefined; +} diff --git a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts index caf6f47be45ce5..56b3b92d287d46 100644 --- a/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/apm/client/apm_synthtrace_kibana_client.ts @@ -10,6 +10,7 @@ import fetch from 'node-fetch'; import pRetry from 'p-retry'; import { Logger } from '../../utils/create_logger'; import { kibanaHeaders } from '../../shared/client_headers'; +import { getFetchAgent } from '../../../cli/utils/ssl'; export class ApmSynthtraceKibanaClient { private readonly logger: Logger; @@ -34,6 +35,7 @@ export class ApmSynthtraceKibanaClient { const response = await fetch(url, { method: 'GET', headers: kibanaHeaders(), + agent: getFetchAgent(url), }); const responseJson = await response.json(); @@ -62,6 +64,7 @@ export class ApmSynthtraceKibanaClient { method: 'POST', headers: kibanaHeaders(), body: '{"force":true}', + agent: getFetchAgent(url), }); if (!res.ok) { @@ -109,6 +112,7 @@ export class ApmSynthtraceKibanaClient { method: 'DELETE', headers: kibanaHeaders(), body: '{"force":true}', + agent: getFetchAgent(url), }); if (!res.ok) { diff --git a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts index c1ac555276a66f..b39efada2abff2 100644 --- a/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/infra/infra_synthtrace_kibana_client.ts @@ -11,6 +11,7 @@ import fetch from 'node-fetch'; import pRetry from 'p-retry'; import { Logger } from '../utils/create_logger'; import { kibanaHeaders } from '../shared/client_headers'; +import { getFetchAgent } from '../../cli/utils/ssl'; export class InfraSynthtraceKibanaClient { private readonly logger: Logger; @@ -30,6 +31,7 @@ export class InfraSynthtraceKibanaClient { const response = await fetch(fleetPackageApiUrl, { method: 'GET', headers: kibanaHeaders(), + agent: getFetchAgent(fleetPackageApiUrl), }); const responseJson = await response.json(); @@ -54,6 +56,7 @@ export class InfraSynthtraceKibanaClient { method: 'POST', headers: kibanaHeaders(), body: '{"force":true}', + agent: getFetchAgent(url), }); }); From f46b6eb823f14d86cef186cfe387097bf39570b6 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Jul 2024 10:30:08 +0200 Subject: [PATCH 27/31] tweak function API --- ...analysis.ts => fetch_log_rate_analysis.ts} | 67 +++++++++++-------- x-pack/plugins/aiops_api/common/types.ts | 2 +- .../get_aiops_log_rate_analysis_function.ts | 18 ++--- .../route_handler_factory.ts | 24 ++++--- .../get_log_rate_analysis/index.ts | 44 ++++++++++++ .../index.ts | 24 ++++--- 6 files changed, 123 insertions(+), 56 deletions(-) rename x-pack/packages/ml/aiops_log_rate_analysis/queries/{fetch_simple_log_rate_analysis.ts => fetch_log_rate_analysis.ts} (81%) create mode 100644 x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis/index.ts diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts similarity index 81% rename from x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts rename to x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts index cd1ca9086f7200..388056665e7c30 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_simple_log_rate_analysis.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts @@ -61,32 +61,45 @@ export interface SimpleSignificantItem { } /** - * Fetches log rate analysis data. + * Asynchronously fetches log rate analysis from an Elasticsearch client. + * Use this function if you want to fetch log rate analysis in other contexts + * than the Log Rate Analysis UI in the ML plugin UI. * - * @param esClient Elasticsearch client. - * @param index The Elasticsearch source index pattern. - * @param start The start of the time range, in Elasticsearch date math, like `now`. - * @param end The end of the time range, in Elasticsearch date math, like `now-24h`. - * @param timefield The Elasticesarch source index pattern time field. - * @param abortSignal Abort signal. - * @param keywordFieldCandidates Optional keyword field candidates. - * @param textFieldCandidates Optional text field candidates. - * @returns Log rate analysis data. + * @param {Object} params - The parameters for fetching log rate analysis. + * @param {ElasticsearchClient} params.esClient - The Elasticsearch client to use for the query. + * @param {AbortSignal} [params.abortSignal] - An optional abort signal to cancel the request. + * @param {Object} params.arguments - The arguments for the log rate analysis query. + * @param {string} params.arguments.index - The index to query against. + * @param {string} params.arguments.start - The start time for the query range. + * @param {string} params.arguments.end - The end time for the query range. + * @param {string} params.arguments.timefield - The field used to filter documents by time. + * @param {string[]} [params.arguments.keywordFieldCandidates] - Optional list of fields to be considered as keyword fields. + * @param {string[]} [params.arguments.textFieldCandidates] - Optional list of fields to be considered as text fields. + * + * @returns {Promise} A promise that resolves when the operation is complete. */ -export const fetchSimpleLogRateAnalysis = async ( - esClient: ElasticsearchClient, - index: string, - start: string, - end: string, - timefield: string, - abortSignal?: AbortSignal, - keywordFieldCandidates: string[] = [], - textFieldCandidates: string[] = [] -) => { +export const fetchLogRateAnalysis = async ({ + esClient, + abortSignal, + arguments: args, +}: { + esClient: ElasticsearchClient; + abortSignal?: AbortSignal; + arguments: { + index: string; + start: string; + end: string; + timefield: string; + keywordFieldCandidates?: string[]; + textFieldCandidates?: string[]; + }; +}) => { const debugStartTime = Date.now(); - const earliestMs = dateMath.parse(start)?.valueOf(); - const latestMs = dateMath.parse(end, { roundUp: true })?.valueOf(); + const earliestMs = dateMath.parse(args.start)?.valueOf(); + const latestMs = dateMath.parse(args.end, { roundUp: true })?.valueOf(); + + const { keywordFieldCandidates = [], textFieldCandidates = [] } = args; if (earliestMs === undefined || latestMs === undefined) { throw new Error('Could not parse time range'); @@ -94,7 +107,7 @@ export const fetchSimpleLogRateAnalysis = async ( const searchQuery = { range: { - [timefield]: { + [args.timefield]: { gte: earliestMs, lte: latestMs, format: 'epoch_millis', @@ -105,10 +118,10 @@ export const fetchSimpleLogRateAnalysis = async ( // CHANGE POINT DETECTION const [error, resp] = await fetchChangePointDetection( esClient, - index, + args.index, earliestMs, latestMs, - timefield, + args.timefield, searchQuery, abortSignal ); @@ -126,11 +139,11 @@ export const fetchSimpleLogRateAnalysis = async ( keywordFieldCandidates.length === 0 && textFieldCandidates.length === 0; const indexInfoParams: AiopsLogRateAnalysisSchema = { - index, + index: args.index, start: earliestMs, end: latestMs, searchQuery: JSON.stringify(searchQuery), - timeFieldName: timefield, + timeFieldName: args.timefield, ...windowParameters, }; diff --git a/x-pack/plugins/aiops_api/common/types.ts b/x-pack/plugins/aiops_api/common/types.ts index 97d29635d1038e..c70578acff8da6 100644 --- a/x-pack/plugins/aiops_api/common/types.ts +++ b/x-pack/plugins/aiops_api/common/types.ts @@ -8,7 +8,7 @@ import type { LogRateChange, SimpleSignificantItem, -} from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; +} from '@kbn/aiops-log-rate-analysis/queries/fetch_log_rate_analysis'; export interface GetAiopsLogRateAnalysisFunctionResponse { content: diff --git a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts index 0b6829b52b7e12..a6882afb0c299e 100644 --- a/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts +++ b/x-pack/plugins/aiops_api/server/assistant_functions/get_aiops_log_rate_analysis_function.ts @@ -10,7 +10,7 @@ import dedent from 'dedent'; import rison from '@kbn/rison'; import { i18n } from '@kbn/i18n'; -import { fetchSimpleLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; +import { fetchLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_log_rate_analysis'; import type { GetAiopsLogRateAnalysisFunctionResponse } from '../../common/types'; @@ -79,14 +79,16 @@ export function registerGetAiopsLogRateAnalysisFunction({ async ({ arguments: args }, abortSignal): Promise => { const { esClient } = resources; - const resp = await fetchSimpleLogRateAnalysis( + const resp = await fetchLogRateAnalysis({ esClient, - args.index, - args.start, - args.end, - args.timefield, - abortSignal - ); + abortSignal, + arguments: { + index: args.index, + start: args.start, + end: args.end, + timefield: args.timefield, + }, + }); const { logRateChange, significantItems, dateHistogramBuckets, windowParameters } = resp; diff --git a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts index 4620580b847482..8363521590d7bc 100644 --- a/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts +++ b/x-pack/plugins/aiops_api/server/routes/log_rate_analysis/route_handler_factory.ts @@ -21,7 +21,7 @@ import { } from '@kbn/aiops-common/constants'; // import { AIOPS_API_ENDPOINT } from '@kbn/aiops-common/constants'; // import { isRequestAbortedError } from '@kbn/aiops-common/is_request_aborted_error'; -import { fetchSimpleLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; +import { fetchLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_log_rate_analysis'; // import { trackAIOpsRouteUsage } from '../../lib/track_route_usage'; import type { AiopsApiLicense } from '../../types'; @@ -55,7 +55,7 @@ export function routeHandlerFactory( return response.forbidden(); } - const client = (await context.core).elasticsearch.client.asCurrentUser; + const esClient = (await context.core).elasticsearch.client.asCurrentUser; const executionContext = createExecutionContext( coreStart, AIOPS_API_PLUGIN_ID, @@ -76,16 +76,18 @@ export function routeHandlerFactory( request.body; return await coreStart.executionContext.withContext(executionContext, async () => { - const logRateAnalysis = await fetchSimpleLogRateAnalysis( - client, - index, - start, - end, - timefield, + const logRateAnalysis = await fetchLogRateAnalysis({ + esClient, abortSignal, - keywordFieldCandidates, - textFieldCandidates - ); + arguments: { + index, + start, + end, + timefield, + keywordFieldCandidates, + textFieldCandidates, + }, + }); return response.ok({ body: logRateAnalysis, diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis/index.ts new file mode 100644 index 00000000000000..f1b03b833f1b04 --- /dev/null +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_rate_analysis/index.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import type { CoreRequestHandlerContext } from '@kbn/core/server'; +import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server'; +import { fetchLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_log_rate_analysis'; +import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client'; + +export async function getLogRateAnalysis({ + apmEventClient, + esClient, + coreContext, + arguments: args, +}: { + apmEventClient: APMEventClient; + esClient: ElasticsearchClient; + coreContext: Pick; + arguments: { + start: string; + end: string; + entities: { + 'service.name'?: string; + 'host.name'?: string; + 'container.id'?: string; + 'kubernetes.pod.name'?: string; + }; + }; +}): ReturnType { + const index = await coreContext.uiSettings.client.get(aiAssistantLogsIndexPattern); + return await fetchLogRateAnalysis({ + esClient, + arguments: { + index, + start: args.start, + end: args.end, + timefield: '@timestamp', + }, + }); +} diff --git a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts index f34936c0a1c592..06bee9de3c9ab8 100644 --- a/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts +++ b/x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts @@ -12,8 +12,6 @@ import type { } from '@kbn/observability-plugin/server/services'; import moment from 'moment'; import { isEmpty } from 'lodash'; -import { fetchSimpleLogRateAnalysis } from '@kbn/aiops-log-rate-analysis/queries/fetch_simple_log_rate_analysis'; -import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server'; import { SERVICE_NAME, SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../../common/es_fields/apm'; import { getApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client'; import { getApmEventClient } from '../../../lib/helpers/get_apm_event_client'; @@ -24,6 +22,7 @@ import { APMDownstreamDependency, getAssistantDownstreamDependencies, } from '../get_apm_downstream_dependencies'; +import { getLogRateAnalysis } from '../get_log_rate_analysis'; import { getLogCategories, LogCategory } from '../get_log_categories'; import { getAnomalies } from '../get_apm_service_summary/get_anomalies'; import { getServiceNameFromSignals } from './get_service_name_from_signals'; @@ -164,14 +163,21 @@ export const getAlertDetailsContextHandler = ( // log rate analysis dataFetchers.push(async () => { - const index = await coreContext.uiSettings.client.get(aiAssistantLogsIndexPattern); - const logRateAnalysis = await fetchSimpleLogRateAnalysis( + const logRateAnalysis = await getLogRateAnalysis({ + apmEventClient, esClient, - index, - moment(alertStartedAt).subtract(15, 'minute').toISOString(), - moment(alertStartedAt).add(15, 'minute').toISOString(), - '@timestamp' - ); + coreContext, + arguments: { + start: moment(alertStartedAt).subtract(15, 'minute').toISOString(), + end: moment(alertStartedAt).add(15, 'minute').toISOString(), + entities: { + 'service.name': serviceName, + 'host.name': hostName, + 'container.id': containerId, + 'kubernetes.pod.name': kubernetesPodName, + }, + }, + }); return { key: 'logRateAnalysis', From 3fb9adb2d2502350474be93976ef8562244044f5 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Jul 2024 11:16:47 +0200 Subject: [PATCH 28/31] tweak function API --- .../queries/fetch_index_info.ts | 3 ++- .../queries/fetch_log_rate_analysis.ts | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts index 7102329dc2a1bc..7bffc512057f6a 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_index_info.ts @@ -48,9 +48,10 @@ export const fetchIndexInfo = async ({ abortSignal?: AbortSignal; arguments: AiopsLogRateAnalysisSchema & { textFieldCandidatesOverrides?: string[]; + includeFieldCandidates?: boolean; }; }): Promise => { - const { textFieldCandidatesOverrides = [], ...params } = args; + const { textFieldCandidatesOverrides = [], includeFieldCandidates = true, ...params } = args; const { index } = params; const allFieldNames: string[] = []; diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts index 388056665e7c30..c785d35b1ec41a 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts @@ -147,13 +147,15 @@ export const fetchLogRateAnalysis = async ({ ...windowParameters, }; - const indexInfo = await fetchIndexInfo( + const indexInfo = await fetchIndexInfo({ esClient, - indexInfoParams, - ['message', 'error.message'], abortSignal, - includeFieldCandidates - ); + arguments: { + ...indexInfoParams, + textFieldCandidatesOverrides: ['message', 'error.message'], + includeFieldCandidates, + }, + }); const baselineNumBuckets = (windowParameters.baselineMax - windowParameters.baselineMin) / intervalMs; From 29daceef4af118e8fa888613b1f026b0df276b4b Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Jul 2024 11:28:52 +0200 Subject: [PATCH 29/31] tweak function API for fetchSignificantTermPValues --- .../queries/fetch_log_rate_analysis.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts index c785d35b1ec41a..5d4bbecc66d0dc 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts @@ -200,17 +200,16 @@ export const fetchLogRateAnalysis = async ({ const pValuesQueue = queue(async function (payload: QueueFieldCandidate) { if (isKeywordFieldCandidate(payload)) { const { keywordFieldCandidate } = payload; - let pValues: Awaited> = []; - pValues = await fetchSignificantTermPValues( + const pValues = await fetchSignificantTermPValues({ esClient, - params, - [keywordFieldCandidate], - undefined, - sampleProbability, - undefined, - abortSignal - ); + abortSignal, + arguments: { + ...params, + fieldNames: [keywordFieldCandidate], + sampleProbability, + }, + }); if (pValues.length > 0) { pValues.forEach((d) => { From cd308c9e42577428e22ce4cf58f25b4afd601567 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Jul 2024 11:35:54 +0200 Subject: [PATCH 30/31] tweak function API for fetchSignificantCategories --- .../queries/fetch_log_rate_analysis.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts index 5d4bbecc66d0dc..57900f32be1d63 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_log_rate_analysis.ts @@ -220,15 +220,15 @@ export const fetchLogRateAnalysis = async ({ } else if (isTextFieldCandidate(payload)) { const { textFieldCandidate } = payload; - const significantCategoriesForField = await fetchSignificantCategories( + const significantCategoriesForField = await fetchSignificantCategories({ esClient, - params, - [textFieldCandidate], - undefined, - sampleProbability, - undefined, - abortSignal - ); + abortSignal, + arguments: { + ...params, + fieldNames: [textFieldCandidate], + sampleProbability, + }, + }); if (significantCategoriesForField.length > 0) { significantCategories.push(...significantCategoriesForField); From f35ca173fedcf0992d080ed660db14b421f87511 Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Fri, 5 Jul 2024 13:40:31 +0200 Subject: [PATCH 31/31] tweak function API for topItemsHandler/fetchTopCategories --- .../queries/fetch_top_categories.ts | 27 ++++++++++++------- .../analysis_handlers/top_items_handler.ts | 16 ++++++----- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_top_categories.ts b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_top_categories.ts index a9fb020c10c516..b8f0ee71d086bd 100644 --- a/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_top_categories.ts +++ b/x-pack/packages/ml/aiops_log_rate_analysis/queries/fetch_top_categories.ts @@ -15,16 +15,25 @@ import type { AiopsLogRateAnalysisSchema } from '../api/schema'; import { fetchCategories } from './fetch_categories'; -export const fetchTopCategories = async ( - esClient: ElasticsearchClient, - params: AiopsLogRateAnalysisSchema, - fieldNames: string[], - logger: Logger, +export const fetchTopCategories = async ({ + esClient, + abortSignal, + emitError, + logger, + arguments: args, +}: { + esClient: ElasticsearchClient; + logger: Logger; + emitError: (m: string) => void; + abortSignal?: AbortSignal; + arguments: AiopsLogRateAnalysisSchema & { + fieldNames: string[]; + sampleProbability?: number; + }; +}) => { // The default value of 1 means no sampling will be used - sampleProbability: number = 1, - emitError: (m: string) => void, - abortSignal?: AbortSignal -) => { + const { fieldNames, sampleProbability = 1, ...params } = args; + const categoriesOverall = await fetchCategories( esClient, params, diff --git a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts index 882057bc12cbf4..5870c92566d11a 100644 --- a/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts +++ b/x-pack/plugins/aiops/server/routes/log_rate_analysis/analysis_handlers/top_items_handler.ts @@ -63,15 +63,17 @@ export const topItemsHandlerFactory = // Get categories of text fields if (textFieldCandidates.length > 0) { topCategories.push( - ...(await fetchTopCategories( + ...(await fetchTopCategories({ esClient, - requestBody, - textFieldCandidates, logger, - stateHandler.sampleProbability(), - responseStream.pushError, - abortSignal - )) + emitError: responseStream.pushError, + abortSignal, + arguments: { + ...requestBody, + fieldNames: textFieldCandidates, + sampleProbability: stateHandler.sampleProbability(), + }, + })) ); if (topCategories.length > 0) {