[ML] AIOps: Add AI Assistant contextual insight to Log Rate Analysis …

…page in ML plugin in Observability serverless. (#186509)

## Summary

Part of #181111.

This adds O11y AI Assistant's contextual insight to the Log Rate
Analysis page in the ML plugin for O11y serverless projects..

Note the code is almost an exact copy of what's currently used in the
O11y UI itself to render the contextual insight when log rate analysis
gets embedded on alert details pages. As a follow up we will be able to
consolidate some code.

[aiops-ai-assistant-contextual-insight-0001.webm](https://github.com/elastic/kibana/assets/230104/08269a40-f45a-4eb9-bd4d-a249cdd46266)

To test this you need to set up AI Assistant in your `kibana.dev.yml`:

```yml
xpack.actions.preconfigured:
  my-gen-ai:
    name: Preconfigured Azure OpenAi
    actionTypeId: .gen-ai
    config:
      apiUrl: <YOUR-API-URL>
      apiProvider: 'Azure OpenAI'
    secrets:
      apiKey: <YOUR-API-KEY>
```

### Checklist

- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [x] This was checked for breaking API changes and was [labeled
appropriately](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)

packages/kbn-optimizer/limits.yml

@@ -103,7 +103,7 @@ pageLoadAssetSize:
   maps: 90000
   mapsEms: 26072
   metricsDataAccess: 73287
-  ml: 82210
+  ml: 85000
   mockIdpPlugin: 30000
   monitoring: 80000
   navigation: 37269

x-pack/plugins/aiops/kibana.jsonc

@@ -22,6 +22,7 @@
     ],
     "optionalPlugins": [
       "cases",
+      "observabilityAIAssistant",
       "usageCollection"
     ],
     "requiredBundles": [

x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_app_state.tsx

@@ -38,6 +38,8 @@ export interface LogRateAnalysisAppStateProps {
   savedSearch: SavedSearch | null;
   /** App dependencies */
   appDependencies: AiopsAppDependencies;
+  /** Optional flag to indicate whether to show contextual insights */
+  showContextualInsights?: boolean;
   /** Optional flag to indicate whether kibana is running in serverless */
   showFrozenDataTierChoice?: boolean;
 }
@@ -46,6 +48,7 @@ export const LogRateAnalysisAppState: FC<LogRateAnalysisAppStateProps> = ({
   dataView,
   savedSearch,
   appDependencies,
+  showContextualInsights = false,
   showFrozenDataTierChoice = true,
 }) => {
   if (!dataView) return null;
@@ -69,7 +72,7 @@ export const LogRateAnalysisAppState: FC<LogRateAnalysisAppStateProps> = ({
           <LogRateAnalysisReduxProvider>
             <StorageContextProvider storage={localStorage} storageKeys={AIOPS_STORAGE_KEYS}>
               <DatePickerContextProvider {...datePickerDeps}>
-                <LogRateAnalysisPage />
+                <LogRateAnalysisPage showContextualInsights={showContextualInsights} />
               </DatePickerContextProvider>
             </StorageContextProvider>
           </LogRateAnalysisReduxProvider>

x-pack/plugins/aiops/public/components/log_rate_analysis/log_rate_analysis_page.tsx

@@ -6,12 +6,14 @@
  */
 
 import type { FC } from 'react';
-import React, { useCallback, useEffect, useState } from 'react';
-import { isEqual } from 'lodash';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
+import { isEqual, orderBy } from 'lodash';
 
 import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 import { EuiFlexGroup, EuiFlexItem, EuiPageBody, EuiPageSection, EuiSpacer } from '@elastic/eui';
 
+import { i18n } from '@kbn/i18n';
+import type { Message } from '@kbn/observability-ai-assistant-plugin/public';
 import type { Filter, Query } from '@kbn/es-query';
 import { FilterStateStore } from '@kbn/es-query';
 import { useUrlState, usePageUrlState } from '@kbn/ml-url-state';
@@ -25,6 +27,10 @@ import {
   setInitialAnalysisStart,
   setDocumentCountChartData,
 } from '@kbn/aiops-log-rate-analysis/state';
+import {
+  LOG_RATE_ANALYSIS_TYPE,
+  type LogRateAnalysisType,
+} from '@kbn/aiops-log-rate-analysis/log_rate_analysis_type';
 
 import { useDataSource } from '../../hooks/use_data_source';
 import { useAiopsAppContext } from '../../hooks/use_aiops_app_context';
@@ -40,10 +46,26 @@ import {
 import { SearchPanel } from '../search_panel';
 import { PageHeader } from '../page_header';
 
+import type { LogRateAnalysisResultsData } from './log_rate_analysis_results';
+
 import { LogRateAnalysisContent } from './log_rate_analysis_content/log_rate_analysis_content';
 
-export const LogRateAnalysisPage: FC = () => {
-  const { data: dataService } = useAiopsAppContext();
+interface SignificantFieldValue {
+  field: string;
+  value: string | number;
+  docCount: number;
+  pValue: number | null;
+}
+
+interface LogRateAnalysisPageProps {
+  showContextualInsights?: boolean;
+}
+
+export const LogRateAnalysisPage: FC<LogRateAnalysisPageProps> = ({
+  showContextualInsights = false,
+}) => {
+  const aiopsAppContext = useAiopsAppContext();
+  const { data: dataService, observabilityAIAssistant } = aiopsAppContext;
   const { dataView, savedSearch } = useDataSource();
 
   const currentSelectedGroup = useCurrentSelectedGroup();
@@ -58,6 +80,15 @@ export const LogRateAnalysisPage: FC = () => {
 
   const [selectedSavedSearch, setSelectedSavedSearch] = useState(savedSearch);
 
+  // Used to store analysis results to be passed on to the AI Assistant.
+  const [logRateAnalysisParams, setLogRateAnalysisParams] = useState<
+    | {
+        logRateAnalysisType: LogRateAnalysisType;
+        significantFieldValues: SignificantFieldValue[];
+      }
+    | undefined
+  >();
+
   useEffect(() => {
     if (savedSearch) {
       setSelectedSavedSearch(savedSearch);
@@ -182,6 +213,88 @@ export const LogRateAnalysisPage: FC = () => {
     }
   };
 
+  const onAnalysisCompleted = (analysisResults: LogRateAnalysisResultsData | undefined) => {
+    const significantFieldValues = orderBy(
+      analysisResults?.significantItems?.map((item) => ({
+        field: item.fieldName,
+        value: item.fieldValue,
+        docCount: item.doc_count,
+        pValue: item.pValue,
+      })),
+      ['pValue', 'docCount'],
+      ['asc', 'asc']
+    ).slice(0, 50);
+
+    const logRateAnalysisType = analysisResults?.analysisType;
+    setLogRateAnalysisParams(
+      significantFieldValues && logRateAnalysisType
+        ? { logRateAnalysisType, significantFieldValues }
+        : undefined
+    );
+  };
+
+  const messages = useMemo<Message[] | undefined>(() => {
+    const hasLogRateAnalysisParams =
+      logRateAnalysisParams && logRateAnalysisParams.significantFieldValues?.length > 0;
+
+    if (!hasLogRateAnalysisParams || !observabilityAIAssistant) {
+      return undefined;
+    }
+
+    const { logRateAnalysisType } = logRateAnalysisParams;
+
+    const header = 'Field name,Field value,Doc count,p-value';
+    const rows = logRateAnalysisParams.significantFieldValues
+      .map((item) => Object.values(item).join(','))
+      .join('\n');
+
+    return observabilityAIAssistant.getContextualInsightMessages({
+      message:
+        'Can you identify possible causes and remediations for these log rate analysis results',
+      instructions: `You are an AIOps expert using Elastic's Kibana on call being consulted about a log rate change that got triggered by a ${logRateAnalysisType} in log messages. Your job is to take immediate action and proceed with both urgency and precision.
+      "Log Rate Analysis" is an AIOps feature that uses advanced statistical methods to identify reasons for increases and decreases in log rates. It makes it easy to find and investigate causes of unusual spikes or dips by using the analysis workflow view.
+      You are using "Log Rate Analysis" and ran the statistical analysis on the log messages which occured during the alert.
+      You received the following analysis results from "Log Rate Analysis" which list statistically significant co-occuring field/value combinations sorted from most significant (lower p-values) to least significant (higher p-values) that ${
+        logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE
+          ? 'contribute to the log rate spike'
+          : 'are less or not present in the log rate dip'
+      }:
+
+      ${
+        logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE
+          ? 'The median log rate in the selected deviation time range is higher than the baseline. Therefore, the results shows statistically significant items within the deviation time range that are contributors to the spike. The "doc count" column refers to the amount of documents in the deviation time range.'
+          : 'The median log rate in the selected deviation time range is lower than the baseline. Therefore, the analysis results table shows statistically significant items within the baseline time range that are less in number or missing within the deviation time range. The "doc count" column refers to the amount of documents in the baseline time range.'
+      }
+
+      ${header}
+      ${rows}
+
+      Based on the above analysis results and your observability expert knowledge, output the following:
+      Analyse the type of these logs and explain their usual purpose (1 paragraph).
+      ${
+        logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE
+          ? 'Based on the type of these logs do a root cause analysis on why the field and value combinations from the analysis results are causing this log rate spike (2 parapraphs)'
+          : 'Based on the type of these logs explain why the statistically significant field and value combinations are less in number or missing from the log rate dip with concrete examples based on the analysis results data which contains items that are present in the baseline time range and are missing or less in number in the deviation time range (2 paragraphs)'
+      }.
+      ${
+        logRateAnalysisType === LOG_RATE_ANALYSIS_TYPE.SPIKE
+          ? 'Recommend concrete remediations to resolve the root cause (3 bullet points).'
+          : ''
+      }
+
+      Do not mention individual p-values from the analysis results.
+      Do not repeat the full list of field names and field values back to the user.
+      Do not repeat the given instructions in your output.`,
+    });
+  }, [logRateAnalysisParams, observabilityAIAssistant]);
+
+  const logRateAnalysisTitle = i18n.translate(
+    'xpack.aiops.observabilityAIAssistantContextualInsight.logRateAnalysisTitle',
+    {
+      defaultMessage: 'Possible causes and remediations',
+    }
+  );
+
   return (
     <EuiPageBody data-test-subj="aiopsLogRateAnalysisPage" paddingSize="none" panelled={false}>
       <PageHeader />
@@ -196,11 +309,24 @@ export const LogRateAnalysisPage: FC = () => {
               setSearchParams={setSearchParams}
             />
           </EuiFlexItem>
-          <LogRateAnalysisContent
-            embeddingOrigin={AIOPS_TELEMETRY_ID.AIOPS_DEFAULT_SOURCE}
-            esSearchQuery={searchQuery}
-            onWindowParametersChange={onWindowParametersHandler}
-          />
+          <EuiFlexItem>
+            <LogRateAnalysisContent
+              embeddingOrigin={AIOPS_TELEMETRY_ID.AIOPS_DEFAULT_SOURCE}
+              esSearchQuery={searchQuery}
+              onWindowParametersChange={onWindowParametersHandler}
+              onAnalysisCompleted={onAnalysisCompleted}
+            />
+          </EuiFlexItem>
+          {showContextualInsights &&
+          observabilityAIAssistant?.ObservabilityAIAssistantContextualInsight &&
+          messages ? (
+            <EuiFlexItem grow={false}>
+              <observabilityAIAssistant.ObservabilityAIAssistantContextualInsight
+                title={logRateAnalysisTitle}
+                messages={messages}
+              />
+            </EuiFlexItem>
+          ) : null}
         </EuiFlexGroup>
       </EuiPageSection>
     </EuiPageBody>

x-pack/plugins/aiops/public/hooks/use_aiops_app_context.ts

@@ -7,6 +7,7 @@
 
 import { createContext, type FC, type PropsWithChildren, useContext } from 'react';
 
+import type { ObservabilityAIAssistantPublicStart } from '@kbn/observability-ai-assistant-plugin/public';
 import type { IStorageWrapper } from '@kbn/kibana-utils-plugin/public';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
@@ -134,6 +135,8 @@ export interface AiopsAppDependencies {
   isServerless?: boolean;
   /** Identifier to indicate the plugin utilizing the component */
   embeddingOrigin?: string;
+  /** Observability AI Assistant */
+  observabilityAIAssistant?: ObservabilityAIAssistantPublicStart;
 }
 
 /**

x-pack/plugins/aiops/public/types/index.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { ObservabilityAIAssistantPublicStart } from '@kbn/observability-ai-assistant-plugin/public';
 import type { ChartsPluginStart } from '@kbn/charts-plugin/public';
 import type { ExecutionContextStart } from '@kbn/core-execution-context-browser';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
@@ -43,6 +44,7 @@ export interface AiopsPluginStartDeps {
   licensing: LicensingPluginStart;
   executionContext: ExecutionContextStart;
   embeddable: EmbeddableStart;
+  observabilityAIAssistant?: ObservabilityAIAssistantPublicStart;
   usageCollection: UsageCollectionSetup;
 }
 

x-pack/plugins/aiops/tsconfig.json

@@ -77,6 +77,7 @@
     "@kbn/unified-search-plugin",
     "@kbn/usage-collection-plugin",
     "@kbn/utility-types",
+    "@kbn/observability-ai-assistant-plugin",
   ],
   "exclude": [
     "target/**/*",

x-pack/plugins/ml/kibana.jsonc

@@ -41,6 +41,7 @@
       "maps",
       "security",
       "spaces",
+      "observabilityAIAssistant",
       "usageCollection",
       "cases"
     ],

x-pack/plugins/ml/public/application/aiops/log_rate_analysis.tsx

@@ -19,7 +19,7 @@ import { useEnabledFeatures } from '../contexts/ml';
 
 export const LogRateAnalysisPage: FC = () => {
   const { services } = useMlKibana();
-  const { showNodeInfo } = useEnabledFeatures();
+  const { showContextualInsights, showNodeInfo } = useEnabledFeatures();
 
   const { selectedDataView: dataView, selectedSavedSearch: savedSearch } = useDataSource();
 
@@ -35,6 +35,7 @@ export const LogRateAnalysisPage: FC = () => {
         <LogRateAnalysis
           dataView={dataView}
           savedSearch={savedSearch}
+          showContextualInsights={showContextualInsights}
           showFrozenDataTierChoice={showNodeInfo}
           appDependencies={pick(services, [
             'analytics',
@@ -53,6 +54,7 @@ export const LogRateAnalysisPage: FC = () => {
             'uiActions',
             'uiSettings',
             'unifiedSearch',
+            'observabilityAIAssistant',
           ])}
         />
       )}

x-pack/plugins/ml/public/application/app.tsx

@@ -86,6 +86,7 @@ const App: FC<AppProps> = ({
       lens: deps.lens,
       licenseManagement: deps.licenseManagement,
       maps: deps.maps,
+      observabilityAIAssistant: deps.observabilityAIAssistant,
       presentationUtil: deps.presentationUtil,
       savedObjectsManagement: deps.savedObjectsManagement,
       savedSearch: deps.savedSearch,

x-pack/plugins/ml/public/application/contexts/kibana/kibana_context.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import type { ObservabilityAIAssistantPublicStart } from '@kbn/observability-ai-assistant-plugin/public';
 import type { DataPublicPluginStart } from '@kbn/data-plugin/public';
 import type { DataViewsPublicPluginStart } from '@kbn/data-views-plugin/public';
 import type { CoreStart } from '@kbn/core/public';
@@ -47,6 +48,7 @@ interface StartPlugins {
   lens: LensPublicStart;
   licenseManagement?: LicenseManagementUIPluginSetup;
   maps?: MapsStartApi;
+  observabilityAIAssistant?: ObservabilityAIAssistantPublicStart;
   presentationUtil: PresentationUtilPluginStart;
   savedObjectsManagement: SavedObjectsManagementPluginStart;
   savedSearch: SavedSearchPublicPluginStart;

x-pack/plugins/ml/public/application/contexts/ml/serverless_context.tsx

@@ -10,6 +10,7 @@ import React, { createContext, useContext, useMemo } from 'react';
 import type { ExperimentalFeatures, MlFeatures } from '../../../../common/constants/app';
 
 export interface EnabledFeatures {
+  showContextualInsights: boolean;
   showNodeInfo: boolean;
   showMLNavMenu: boolean;
   showLicenseInfo: boolean;
@@ -18,13 +19,15 @@ export interface EnabledFeatures {
   isNLPEnabled: boolean;
   showRuleFormV2: boolean;
 }
-export const EnabledFeaturesContext = createContext({
+export const EnabledFeaturesContext = createContext<EnabledFeatures>({
+  showContextualInsights: true,
   showNodeInfo: true,
   showMLNavMenu: true,
   showLicenseInfo: true,
   isADEnabled: true,
   isDFAEnabled: true,
   isNLPEnabled: true,
+  showRuleFormV2: true,
 });
 
 interface Props {
@@ -42,6 +45,7 @@ export const EnabledFeaturesContextProvider: FC<PropsWithChildren<Props>> = ({
   experimentalFeatures,
 }) => {
   const features: EnabledFeatures = {
+    showContextualInsights: isServerless,
     showNodeInfo: !isServerless,
     showMLNavMenu,
     showLicenseInfo: !isServerless,

x-pack/plugins/ml/public/plugin.ts

@@ -16,6 +16,7 @@ import type {
 import { BehaviorSubject, mergeMap } from 'rxjs';
 import { take } from 'rxjs';
 
+import type { ObservabilityAIAssistantPublicStart } from '@kbn/observability-ai-assistant-plugin/public';
 import type { UnifiedSearchPublicPluginStart } from '@kbn/unified-search-plugin/public';
 import type { ManagementSetup } from '@kbn/management-plugin/public';
 import type { LocatorPublic, SharePluginSetup, SharePluginStart } from '@kbn/share-plugin/public';
@@ -88,6 +89,7 @@ export interface MlStartDependencies {
   lens: LensPublicStart;
   licensing: LicensingPluginStart;
   maps?: MapsStartApi;
+  observabilityAIAssistant?: ObservabilityAIAssistantPublicStart;
   presentationUtil: PresentationUtilPluginStart;
   savedObjectsManagement: SavedObjectsManagementPluginStart;
   savedSearch: SavedSearchPublicPluginStart;
@@ -180,6 +182,7 @@ export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
             licensing: pluginsStart.licensing,
             management: pluginsSetup.management,
             maps: pluginsStart.maps,
+            observabilityAIAssistant: pluginsStart.observabilityAIAssistant,
             presentationUtil: pluginsStart.presentationUtil,
             savedObjectsManagement: pluginsStart.savedObjectsManagement,
             savedSearch: pluginsStart.savedSearch,

x-pack/plugins/ml/tsconfig.json

@@ -127,6 +127,7 @@
     "@kbn/react-kibana-context-render",
     "@kbn/esql-utils",
     "@kbn/core-lifecycle-browser",
-    "@kbn/json-schemas",
-  ],
+    "@kbn/observability-ai-assistant-plugin",
+    "@kbn/json-schemas"
+  ]
 }