Add downstream dependency service name to logs and errors to improve alert insights

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_log_categories/index.ts

@@ -9,25 +9,32 @@ import datemath from '@elastic/datemath';
 import { ElasticsearchClient } from '@kbn/core-elasticsearch-server';
 import type { CoreRequestHandlerContext } from '@kbn/core/server';
 import { aiAssistantLogsIndexPattern } from '@kbn/observability-ai-assistant-plugin/server';
+import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
 import {
   SERVICE_NAME,
   CONTAINER_ID,
   HOST_NAME,
   KUBERNETES_POD_NAME,
+  PROCESSOR_EVENT,
+  TRACE_ID,
 } from '../../../../common/es_fields/apm';
 import { getTypedSearch } from '../../../utils/create_typed_es_client';
+import { getDownstreamServiceResource } from '../get_observability_alert_details_context/get_downstream_dependency_name';
 
 export interface LogCategory {
   errorCategory: string;
   docCount: number;
   sampleMessage: string;
+  downstreamServiceResource?: string;
 }
 
 export async function getLogCategories({
+  apmEventClient,
   esClient,
   coreContext,
   arguments: args,
 }: {
+  apmEventClient: APMEventClient;
   esClient: ElasticsearchClient;
   coreContext: Pick<CoreRequestHandlerContext, 'uiSettings'>;
   arguments: {
@@ -55,6 +62,10 @@ export async function getLogCategories({
 
   const query = {
     bool: {
+      must_not: [
+        // exclude APM errors
+        { term: { [PROCESSOR_EVENT]: 'error' } },
+      ],
       filter: [
         ...keyValueFilters,
         { exists: { field: 'message' } },
@@ -70,6 +81,8 @@ export async function getLogCategories({
     },
   };
 
+  console.log('query', JSON.stringify(query));
+
   const hitCountRes = await search({
     index,
     size: 0,
@@ -101,7 +114,7 @@ export async function getLogCategories({
                 top_hits: {
                   sort: { '@timestamp': 'desc' as const },
                   size: 1,
-                  _source: ['message'],
+                  _source: ['message', TRACE_ID],
                 },
               },
             },
@@ -111,12 +124,28 @@ export async function getLogCategories({
     },
   });
 
-  return categorizedLogsRes.aggregations?.sampling.categories?.buckets.map(
-    ({ doc_count: docCount, key, sample }) => {
-      const sampleMessage = (sample.hits.hits[0]._source as { message: string }).message;
-      return { errorCategory: key as string, docCount, sampleMessage };
+  const promises = categorizedLogsRes.aggregations?.sampling.categories?.buckets.map(
+    async ({ doc_count: docCount, key, sample }) => {
+      const hit = sample.hits.hits[0]._source as { message: string; trace?: { id: string } };
+      const sampleMessage = hit?.message;
+      const sampleTraceId = hit?.trace?.id;
+
+      if (!sampleTraceId) {
+        return { errorCategory: key as string, docCount, sampleMessage };
+      }
+
+      const downstreamServiceResource = await getDownstreamServiceResource({
+        traceId: sampleTraceId,
+        start,
+        end,
+        apmEventClient,
+      });
+
+      return { errorCategory: key as string, docCount, sampleMessage, downstreamServiceResource };
     }
   );
+
+  return Promise.all(promises ?? []);
 }
 
 // field/value pairs should match, or the field should not exist

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_apm_errors.ts

@@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import datemath from '@elastic/datemath';
+import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
+import { getErrorGroupMainStatistics } from '../../errors/get_error_groups/get_error_group_main_statistics';
+import { getDownstreamServiceResource } from './get_downstream_dependency_name';
+
+export async function getApmErrors(params: {
+  apmEventClient: APMEventClient;
+  start: string;
+  end: string;
+  serviceName: string;
+  serviceEnvironment?: string;
+}) {
+  const { apmEventClient, serviceName, serviceEnvironment } = params;
+
+  const start = datemath.parse(params.start)?.valueOf()!;
+  const end = datemath.parse(params.end)?.valueOf()!;
+
+  const res = await getErrorGroupMainStatistics({
+    serviceName,
+    apmEventClient,
+    environment: serviceEnvironment,
+    start,
+    end,
+    maxNumberOfErrorGroups: 100,
+  });
+
+  const promises = res.errorGroups.map(async (errorGroup) => {
+    if (!errorGroup.traceId) {
+      return { ...errorGroup, downstreamServiceResource: undefined };
+    }
+
+    const downstreamServiceResource = await getDownstreamServiceResource({
+      traceId: errorGroup.traceId,
+      start,
+      end,
+      apmEventClient,
+    });
+
+    return { ...errorGroup, downstreamServiceResource };
+  });
+
+  return Promise.all(promises);
+}

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/get_downstream_dependency_name.ts

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { rangeQuery } from '@kbn/observability-plugin/server';
+import { ApmDocumentType } from '../../../../common/document_type';
+import { termQuery } from '../../../../common/utils/term_query';
+import {
+  EVENT_OUTCOME,
+  SPAN_DESTINATION_SERVICE_RESOURCE,
+  TRACE_ID,
+} from '../../../../common/es_fields/apm';
+import { APMEventClient } from '../../../lib/helpers/create_es_client/create_apm_event_client';
+import { RollupInterval } from '../../../../common/rollup';
+
+export async function getDownstreamServiceResource({
+  traceId,
+  start,
+  end,
+  apmEventClient,
+}: {
+  traceId: string;
+  start: number;
+  end: number;
+  apmEventClient: APMEventClient;
+}) {
+  const response = await apmEventClient.search('get_error_group_main_statistics', {
+    apm: {
+      sources: [
+        {
+          documentType: ApmDocumentType.SpanEvent,
+          rollupInterval: RollupInterval.None,
+        },
+      ],
+    },
+    body: {
+      track_total_hits: false,
+      size: 1,
+      _source: ['span.destination.service'],
+      query: {
+        bool: {
+          filter: [
+            ...termQuery(TRACE_ID, traceId),
+            ...termQuery(EVENT_OUTCOME, 'failure'),
+            ...rangeQuery(start, end),
+            { exists: { field: SPAN_DESTINATION_SERVICE_RESOURCE } },
+          ],
+        },
+      },
+    },
+  });
+
+  const hit = response.hits.hits[0];
+  return hit?._source?.span.destination?.service.resource;
+}

x-pack/plugins/observability_solution/apm/server/routes/assistant_functions/get_observability_alert_details_context/index.ts

@@ -12,6 +12,7 @@ import {
 } from '@kbn/observability-plugin/server/services';
 import moment from 'moment';
 import { isEmpty } from 'lodash';
+import { SERVICE_NAME, SPAN_DESTINATION_SERVICE_RESOURCE } from '../../../../common/es_fields/apm';
 import { getApmAlertsClient } from '../../../lib/helpers/get_apm_alerts_client';
 import { getApmEventClient } from '../../../lib/helpers/get_apm_event_client';
 import { getMlClient } from '../../../lib/helpers/get_ml_client';
@@ -24,6 +25,7 @@ import { getServiceNameFromSignals } from './get_service_name_from_signals';
 import { getContainerIdFromSignals } from './get_container_id_from_signals';
 import { getExitSpanChangePoints, getServiceChangePoints } from '../get_changepoints';
 import { APMRouteHandlerResources } from '../../apm_routes/register_apm_server_routes';
+import { getApmErrors } from './get_apm_errors';
 
 export const getAlertDetailsContextHandler = (
   resourcePlugins: APMRouteHandlerResources['plugins'],
@@ -134,7 +136,7 @@ export const getAlertDetailsContextHandler = (
             arguments: {
               'service.name': serviceName,
               'service.environment': serviceEnvironment,
-              start: moment(alertStartedAt).subtract(15, 'minute').toISOString(),
+              start: moment(alertStartedAt).subtract(24, 'hours').toISOString(),
               end: alertStartedAt,
             },
           })
@@ -143,6 +145,7 @@ export const getAlertDetailsContextHandler = (
 
     const logCategoriesPromise = handleError(() =>
       getLogCategories({
+        apmEventClient,
         esClient,
         coreContext,
         arguments: {
@@ -156,6 +159,18 @@ export const getAlertDetailsContextHandler = (
       })
     );
 
+    const apmErrorsPromise = serviceName
+      ? handleError(() =>
+          getApmErrors({
+            apmEventClient,
+            start: moment(alertStartedAt).subtract(15, 'minute').toISOString(),
+            end: alertStartedAt,
+            serviceName,
+            serviceEnvironment,
+          })
+        )
+      : undefined;
+
     const serviceChangePointsPromise = handleError(() =>
       getServiceChangePoints({
         apmEventClient,
@@ -192,13 +207,15 @@ export const getAlertDetailsContextHandler = (
       serviceSummary,
       downstreamDependencies,
       logCategories,
+      apmErrors,
       serviceChangePoints,
       exitSpanChangePoints,
       anomalies,
     ] = await Promise.all([
       serviceSummaryPromise,
       downstreamDependenciesPromise,
       logCategoriesPromise,
+      apmErrorsPromise,
       serviceChangePointsPromise,
       exitSpanChangePointsPromise,
       anomaliesPromise,
@@ -207,7 +224,7 @@ export const getAlertDetailsContextHandler = (
     return [
       {
         key: 'serviceSummary',
-        description: 'Metadata for the service where the alert occurred',
+        description: `Metadata for the service "${serviceName}" that produced the alert. The alert might be caused by an issue in the service itself or one of its dependencies.`,
         data: serviceSummary,
       },
       {
@@ -227,14 +244,64 @@ export const getAlertDetailsContextHandler = (
       },
       {
         key: 'logCategories',
-        description: `Log events occurring around the time of the alert`,
-        data: logCategories,
+        description: `Related log events occurring shortly before the alert was triggered.`,
+        data: logCategoriesWithDownstreamServiceName(logCategories, downstreamDependencies),
+      },
+      {
+        key: 'apmErrors',
+        description: `Exceptions for the service "${serviceName}". If a downstream service name is included this could be a possible root cause. If relevant please describe what the error means and what it could be caused by.`,
+        data: apmErrorsWithDownstreamServiceName(apmErrors, downstreamDependencies),
       },
       {
         key: 'anomalies',
-        description: `Anomalies for services running in the environment "${serviceEnvironment}"`,
+        description: `Anomalies for services running in the environment "${serviceEnvironment}". Anomalies are detected using machine learning and can help you spot unusual patterns in your data.`,
         data: anomalies,
       },
     ].filter(({ data }) => !isEmpty(data));
   };
 };
+
+function apmErrorsWithDownstreamServiceName(
+  apmErrors?: Awaited<ReturnType<typeof getApmErrors>>,
+  downstreamDependencies?: Awaited<ReturnType<typeof getAssistantDownstreamDependencies>>
+) {
+  return apmErrors?.map(({ name, lastSeen, occurrences, downstreamServiceResource }) => {
+    const downstreamServiceName = downstreamDependencies?.find(
+      (dependency) => dependency['span.destination.service.resource'] === downstreamServiceResource
+    )?.['service.name'];
+
+    return {
+      message: name,
+      lastSeen: new Date(lastSeen).toISOString(),
+      occurrences,
+      downstream: {
+        [SPAN_DESTINATION_SERVICE_RESOURCE]: downstreamServiceResource,
+        [SERVICE_NAME]: downstreamServiceName,
+      },
+    };
+  });
+}
+
+function logCategoriesWithDownstreamServiceName(
+  logCategories?: Awaited<ReturnType<typeof getLogCategories>>,
+  downstreamDependencies?: Awaited<ReturnType<typeof getAssistantDownstreamDependencies>>
+) {
+  return logCategories?.map(
+    ({ errorCategory, docCount, sampleMessage, downstreamServiceResource }) => {
+      const downstreamServiceName = downstreamDependencies?.find(
+        (dependency) =>
+          dependency['span.destination.service.resource'] === downstreamServiceResource
+      )?.['service.name'];
+
+      return {
+        errorCategory,
+        docCount,
+        sampleMessage,
+        downstream: {
+          [SPAN_DESTINATION_SERVICE_RESOURCE]: downstreamServiceResource,
+          [SERVICE_NAME]: downstreamServiceName,
+        },
+      };
+    }
+  );
+}

x-pack/plugins/observability_solution/apm/server/routes/errors/get_error_groups/get_error_group_main_statistics.ts

@@ -16,6 +16,7 @@ import {
   ERROR_GROUP_NAME,
   ERROR_LOG_MESSAGE,
   SERVICE_NAME,
+  TRACE_ID,
   TRANSACTION_NAME,
   TRANSACTION_TYPE,
 } from '../../../../common/es_fields/apm';
@@ -34,6 +35,7 @@ export interface ErrorGroupMainStatisticsResponse {
     culprit: string | undefined;
     handled: boolean | undefined;
     type: string | undefined;
+    traceId: string | undefined;
   }>;
   maxCountExceeded: boolean;
 }
@@ -52,10 +54,10 @@ export async function getErrorGroupMainStatistics({
   transactionType,
   searchQuery,
 }: {
-  kuery: string;
+  kuery?: string;
   serviceName: string;
   apmEventClient: APMEventClient;
-  environment: string;
+  environment?: string;
   sortField?: string;
   sortDirection?: 'asc' | 'desc';
   start: number;
@@ -124,6 +126,7 @@ export async function getErrorGroupMainStatistics({
               top_hits: {
                 size: 1,
                 _source: [
+                  TRACE_ID,
                   ERROR_LOG_MESSAGE,
                   ERROR_EXC_MESSAGE,
                   ERROR_EXC_HANDLED,
@@ -158,6 +161,7 @@ export async function getErrorGroupMainStatistics({
         culprit: bucket.sample.hits.hits[0]._source.error.culprit,
         handled: bucket.sample.hits.hits[0]._source.error.exception?.[0].handled,
         type: bucket.sample.hits.hits[0]._source.error.exception?.[0].type,
+        traceId: bucket.sample.hits.hits[0]._source.trace?.id,
       };
     }) ?? [];