[7.x] [Logs UI] Log threshold rule performance improvements (#102650)

x-pack/plugins/infra/common/alerting/logs/log_threshold/types.ts

@@ -100,7 +100,7 @@ export enum AlertStates {
   ERROR,
 }
 
-const ThresholdRT = rt.type({
+export const ThresholdRT = rt.type({
   comparator: ComparatorRT,
   value: rt.number,
 });
@@ -240,65 +240,117 @@ const chartPreviewHistogramBucket = rt.type({
   doc_count: rt.number,
 });
 
+const ChartPreviewBucketsRT = rt.partial({
+  histogramBuckets: rt.type({
+    buckets: rt.array(chartPreviewHistogramBucket),
+  }),
+});
+
 // ES query responses //
+const hitsRT = rt.type({
+  total: rt.type({
+    value: rt.number,
+  }),
+});
+
+const bucketFieldsRT = rt.type({
+  key: rt.record(rt.string, rt.string),
+  doc_count: rt.number,
+});
+
+const afterKeyRT = rt.partial({
+  after_key: rt.record(rt.string, rt.string),
+});
+
 export const UngroupedSearchQueryResponseRT = rt.intersection([
   commonSearchSuccessResponseFieldsRT,
   rt.intersection([
     rt.type({
-      hits: rt.type({
-        total: rt.type({
-          value: rt.number,
-        }),
-      }),
+      hits: hitsRT,
     }),
-    // Chart preview buckets
     rt.partial({
-      aggregations: rt.type({
-        histogramBuckets: rt.type({
-          buckets: rt.array(chartPreviewHistogramBucket),
-        }),
-      }),
+      aggregations: ChartPreviewBucketsRT,
     }),
   ]),
 ]);
 
 export type UngroupedSearchQueryResponse = rt.TypeOf<typeof UngroupedSearchQueryResponseRT>;
 
-export const GroupedSearchQueryResponseRT = rt.intersection([
+export const UnoptimizedGroupedSearchQueryResponseRT = rt.intersection([
   commonSearchSuccessResponseFieldsRT,
   rt.type({
     aggregations: rt.type({
       groups: rt.intersection([
         rt.type({
           buckets: rt.array(
             rt.type({
-              key: rt.record(rt.string, rt.string),
-              doc_count: rt.number,
+              ...bucketFieldsRT.props,
               filtered_results: rt.intersection([
                 rt.type({
                   doc_count: rt.number,
                 }),
-                // Chart preview buckets
-                rt.partial({
-                  histogramBuckets: rt.type({
-                    buckets: rt.array(chartPreviewHistogramBucket),
-                  }),
-                }),
+                ChartPreviewBucketsRT,
               ]),
             })
           ),
         }),
-        rt.partial({
-          after_key: rt.record(rt.string, rt.string),
-        }),
+        afterKeyRT,
       ]),
     }),
-    hits: rt.type({
-      total: rt.type({
-        value: rt.number,
-      }),
+    hits: hitsRT,
+  }),
+]);
+
+export type UnoptimizedGroupedSearchQueryResponse = rt.TypeOf<
+  typeof UnoptimizedGroupedSearchQueryResponseRT
+>;
+
+export const OptimizedGroupedSearchQueryResponseRT = rt.intersection([
+  commonSearchSuccessResponseFieldsRT,
+  rt.type({
+    aggregations: rt.type({
+      groups: rt.intersection([
+        rt.type({
+          buckets: rt.array(rt.intersection([bucketFieldsRT, ChartPreviewBucketsRT])),
+        }),
+        afterKeyRT,
+      ]),
     }),
+    hits: hitsRT,
   }),
 ]);
 
+export type OptimizedGroupedSearchQueryResponse = rt.TypeOf<
+  typeof OptimizedGroupedSearchQueryResponseRT
+>;
+
+export const GroupedSearchQueryResponseRT = rt.union([
+  UnoptimizedGroupedSearchQueryResponseRT,
+  OptimizedGroupedSearchQueryResponseRT,
+]);
+
 export type GroupedSearchQueryResponse = rt.TypeOf<typeof GroupedSearchQueryResponseRT>;
+
+export const isOptimizedGroupedSearchQueryResponse = (
+  response: GroupedSearchQueryResponse['aggregations']['groups']['buckets']
+): response is OptimizedGroupedSearchQueryResponse['aggregations']['groups']['buckets'] => {
+  const result = response[0];
+  return result && !result.hasOwnProperty('filtered_results');
+};
+
+export const isOptimizableGroupedThreshold = (
+  selectedComparator: AlertParams['count']['comparator'],
+  selectedValue?: AlertParams['count']['value']
+) => {
+  if (selectedComparator === Comparator.GT) {
+    return true;
+  } else if (
+    typeof selectedValue === 'number' &&
+    selectedComparator === Comparator.GT_OR_EQ &&
+    selectedValue > 0
+  ) {
+    return true;
+  } else {
+    return false;
+  }
+};

x-pack/plugins/infra/common/http_api/log_alerts/chart_preview_data.ts

@@ -7,6 +7,7 @@
 
 import * as rt from 'io-ts';
 import {
+  ThresholdRT,
   countCriteriaRT,
   timeUnitRT,
   timeSizeRT,
@@ -58,6 +59,14 @@ export type GetLogAlertsChartPreviewDataSuccessResponsePayload = rt.TypeOf<
 export const getLogAlertsChartPreviewDataAlertParamsSubsetRT: any = rt.intersection([
   rt.type({
     criteria: countCriteriaRT,
+    count: rt.intersection([
+      rt.type({
+        comparator: ThresholdRT.props.comparator,
+      }),
+      rt.partial({
+        value: ThresholdRT.props.value,
+      }),
+    ]),
     timeUnit: timeUnitRT,
     timeSize: timeSizeRT,
   }),

x-pack/plugins/infra/public/alerting/log_threshold/components/expression_editor/criterion_preview_chart.tsx

@@ -68,6 +68,10 @@ export const CriterionPreview: React.FC<Props> = ({
     const criteria = field && comparator && value ? [{ field, comparator, value }] : [];
     const params = {
       criteria,
+      count: {
+        comparator: alertParams.count.comparator,
+        value: alertParams.count.value,
+      },
       timeSize: alertParams.timeSize,
       timeUnit: alertParams.timeUnit,
       groupBy: alertParams.groupBy,
@@ -78,7 +82,14 @@ export const CriterionPreview: React.FC<Props> = ({
     } catch (error) {
       return null;
     }
-  }, [alertParams.timeSize, alertParams.timeUnit, alertParams.groupBy, chartCriterion]);
+  }, [
+    alertParams.timeSize,
+    alertParams.timeUnit,
+    alertParams.groupBy,
+    alertParams.count.comparator,
+    alertParams.count.value,
+    chartCriterion,
+  ]);
 
   // Check for the existence of properties that are necessary for a meaningful chart.
   if (chartAlertParams === null || chartAlertParams.criteria.length === 0) return null;

x-pack/plugins/infra/public/alerting/log_threshold/components/expression_editor/editor.tsx

@@ -23,6 +23,7 @@ import {
   PartialRatioAlertParams,
   ThresholdType,
   timeUnitRT,
+  isOptimizableGroupedThreshold,
 } from '../../../../../common/alerting/logs/log_threshold/types';
 import { decodeOrThrow } from '../../../../../common/runtime_types';
 import { ObjectEntries } from '../../../../../common/utility_types';
@@ -255,6 +256,15 @@ export const Editor: React.FC<
     setHasSetDefaults(true);
   });
 
+  const shouldShowGroupByOptimizationWarning = useMemo(() => {
+    const hasSetGroupBy = alertParams.groupBy && alertParams.groupBy.length > 0;
+    return (
+      hasSetGroupBy &&
+      alertParams.count &&
+      !isOptimizableGroupedThreshold(alertParams.count.comparator, alertParams.count.value)
+    );
+  }, [alertParams]);
+
   // Wait until the alert param defaults have been set
   if (!hasSetDefaults) return null;
 
@@ -299,6 +309,21 @@ export const Editor: React.FC<
 
       {alertParams.criteria && isRatioAlert(alertParams.criteria) && criteriaComponent}
 
+      {shouldShowGroupByOptimizationWarning && (
+        <>
+          <EuiSpacer size="l" />
+          <EuiCallOut color="warning">
+            {i18n.translate('xpack.infra.logs.alertFlyout.groupByOptimizationWarning', {
+              defaultMessage:
+                'When setting a "group by" we highly recommend using the "{comparator}" comparator for your threshold. This can lead to significant performance improvements.',
+              values: {
+                comparator: Comparator.GT,
+              },
+            })}
+          </EuiCallOut>
+        </>
+      )}
+
       <EuiSpacer size="l" />
     </>
   );

x-pack/plugins/infra/server/lib/alerting/log_threshold/log_threshold_chart_preview.ts

@@ -23,6 +23,7 @@ import {
   UngroupedSearchQueryResponse,
   GroupedSearchQueryResponse,
   GroupedSearchQueryResponseRT,
+  isOptimizedGroupedSearchQueryResponse,
 } from '../../../../common/alerting/logs/log_threshold/types';
 import { decodeOrThrow } from '../../../../common/runtime_types';
 import { ResolvedLogSourceConfiguration } from '../../../../common/log_sources';
@@ -97,10 +98,19 @@ const addHistogramAggregationToQuery = (
   };
 
   if (isGrouped) {
-    query.body.aggregations.groups.aggregations.filtered_results = {
-      ...query.body.aggregations.groups.aggregations.filtered_results,
-      aggregations: histogramAggregation,
-    };
+    const isOptimizedQuery = !query.body.aggregations.groups.aggregations?.filtered_results;
+
+    if (isOptimizedQuery) {
+      query.body.aggregations.groups.aggregations = {
+        ...query.body.aggregations.groups.aggregations,
+        ...histogramAggregation,
+      };
+    } else {
+      query.body.aggregations.groups.aggregations.filtered_results = {
+        ...query.body.aggregations.groups.aggregations.filtered_results,
+        aggregations: histogramAggregation,
+      };
+    }
   } else {
     query.body = {
       ...query.body,
@@ -151,18 +161,34 @@ const getGroupedResults = async (
 const processGroupedResults = (
   results: GroupedSearchQueryResponse['aggregations']['groups']['buckets']
 ): Series => {
-  return results.reduce<Series>((series, group) => {
-    if (!group.filtered_results.histogramBuckets) return series;
-    const groupName = Object.values(group.key).join(', ');
-    const points = group.filtered_results.histogramBuckets.buckets.reduce<Point[]>(
-      (pointsAcc, bucket) => {
+  const getGroupName = (
+    key: GroupedSearchQueryResponse['aggregations']['groups']['buckets'][0]['key']
+  ) => Object.values(key).join(', ');
+
+  if (isOptimizedGroupedSearchQueryResponse(results)) {
+    return results.reduce<Series>((series, group) => {
+      if (!group.histogramBuckets) return series;
+      const groupName = getGroupName(group.key);
+      const points = group.histogramBuckets.buckets.reduce<Point[]>((pointsAcc, bucket) => {
         const { key, doc_count: count } = bucket;
         return [...pointsAcc, { timestamp: key, value: count }];
-      },
-      []
-    );
-    return [...series, { id: groupName, points }];
-  }, []);
+      }, []);
+      return [...series, { id: groupName, points }];
+    }, []);
+  } else {
+    return results.reduce<Series>((series, group) => {
+      if (!group.filtered_results.histogramBuckets) return series;
+      const groupName = getGroupName(group.key);
+      const points = group.filtered_results.histogramBuckets.buckets.reduce<Point[]>(
+        (pointsAcc, bucket) => {
+          const { key, doc_count: count } = bucket;
+          return [...pointsAcc, { timestamp: key, value: count }];
+        },
+        []
+      );
+      return [...series, { id: groupName, points }];
+    }, []);
+  }
 };
 
 const processUngroupedResults = (results: UngroupedSearchQueryResponse): Series => {