[Logs UI] Log threshold rule performance improvements

x-pack/plugins/infra/common/alerting/logs/log_threshold/types.ts

@@ -100,7 +100,7 @@ export enum AlertStates {
   ERROR,
 }
 
-const ThresholdRT = rt.type({
+export const ThresholdRT = rt.type({
   comparator: ComparatorRT,
   value: rt.number,
 });
@@ -264,7 +264,7 @@ export const UngroupedSearchQueryResponseRT = rt.intersection([
 
 export type UngroupedSearchQueryResponse = rt.TypeOf<typeof UngroupedSearchQueryResponseRT>;
 
-export const GroupedSearchQueryResponseRT = rt.intersection([
+export const UnoptimizedGroupedSearchQueryResponseRT = rt.intersection([
   commonSearchSuccessResponseFieldsRT,
   rt.type({
     aggregations: rt.type({
@@ -301,4 +301,58 @@ export const GroupedSearchQueryResponseRT = rt.intersection([
   }),
 ]);
 
+export type UnoptimizedGroupedSearchQueryResponse = rt.TypeOf<
+  typeof UnoptimizedGroupedSearchQueryResponseRT
+>;
+
+export const OptimizedGroupedSearchQueryResponseRT = rt.intersection([
+  commonSearchSuccessResponseFieldsRT,
+  rt.type({
+    aggregations: rt.type({
+      groups: rt.intersection([
+        rt.type({
+          buckets: rt.array(
+            rt.intersection([
+              rt.type({
+                key: rt.record(rt.string, rt.string),
+                doc_count: rt.number,
+              }),
+              // Chart preview buckets
+              rt.partial({
+                histogramBuckets: rt.type({
+                  buckets: rt.array(chartPreviewHistogramBucket),
+                }),
+              }),
+            ])
+          ),
+        }),
+        rt.partial({
+          after_key: rt.record(rt.string, rt.string),
+        }),
+      ]),
+    }),
+    hits: rt.type({
+      total: rt.type({
+        value: rt.number,
+      }),
+    }),
+  }),
+]);
+
+export type OptimizedGroupedSearchQueryResponse = rt.TypeOf<
+  typeof OptimizedGroupedSearchQueryResponseRT
+>;
+
+export const GroupedSearchQueryResponseRT = rt.union([
+  UnoptimizedGroupedSearchQueryResponseRT,
+  OptimizedGroupedSearchQueryResponseRT,
+]);
+
 export type GroupedSearchQueryResponse = rt.TypeOf<typeof GroupedSearchQueryResponseRT>;
+
+export const isOptimizedGroupedSearchQueryResponse = (
+  response: GroupedSearchQueryResponse['aggregations']['groups']['buckets']
+): response is OptimizedGroupedSearchQueryResponse['aggregations']['groups']['buckets'] => {
+  const result = response[0];
+  return result && !result.hasOwnProperty('filtered_results');
+};

x-pack/plugins/infra/common/http_api/log_alerts/chart_preview_data.ts

@@ -7,6 +7,7 @@
 
 import * as rt from 'io-ts';
 import {
+  ThresholdRT,
   countCriteriaRT,
   timeUnitRT,
   timeSizeRT,
@@ -58,6 +59,9 @@ export type GetLogAlertsChartPreviewDataSuccessResponsePayload = rt.TypeOf<
 export const getLogAlertsChartPreviewDataAlertParamsSubsetRT: any = rt.intersection([
   rt.type({
     criteria: countCriteriaRT,
+    count: rt.type({
+      comparator: ThresholdRT.props.comparator,
+    }),
     timeUnit: timeUnitRT,
     timeSize: timeSizeRT,
   }),

x-pack/plugins/infra/public/alerting/log_threshold/components/expression_editor/criterion_preview_chart.tsx

@@ -68,6 +68,9 @@ export const CriterionPreview: React.FC<Props> = ({
     const criteria = field && comparator && value ? [{ field, comparator, value }] : [];
     const params = {
       criteria,
+      count: {
+        comparator: alertParams.count.comparator,
+      },
       timeSize: alertParams.timeSize,
       timeUnit: alertParams.timeUnit,
       groupBy: alertParams.groupBy,
@@ -78,7 +81,13 @@ export const CriterionPreview: React.FC<Props> = ({
     } catch (error) {
       return null;
     }
-  }, [alertParams.timeSize, alertParams.timeUnit, alertParams.groupBy, chartCriterion]);
+  }, [
+    alertParams.timeSize,
+    alertParams.timeUnit,
+    alertParams.groupBy,
+    alertParams.count.comparator,
+    chartCriterion,
+  ]);
 
   // Check for the existence of properties that are necessary for a meaningful chart.
   if (chartAlertParams === null || chartAlertParams.criteria.length === 0) return null;

x-pack/plugins/infra/public/alerting/log_threshold/components/expression_editor/editor.tsx

@@ -255,6 +255,13 @@ export const Editor: React.FC<
     setHasSetDefaults(true);
   });
 
+  const shouldShowGroupByOptimizationWarning = useMemo(() => {
+    const hasSetGroupBy = alertParams.groupBy && alertParams.groupBy.length > 0;
+    const hasSetOptimizableThresholdComparator =
+      alertParams.count && alertParams.count.comparator === Comparator.GT;
+    return hasSetGroupBy && !hasSetOptimizableThresholdComparator;
+  }, [alertParams]);
+
   // Wait until the alert param defaults have been set
   if (!hasSetDefaults) return null;
 
@@ -299,6 +306,21 @@ export const Editor: React.FC<
 
       {alertParams.criteria && isRatioAlert(alertParams.criteria) && criteriaComponent}
 
+      {shouldShowGroupByOptimizationWarning && (
+        <>
+          <EuiSpacer size="l" />
+          <EuiCallOut color="warning">
+            {i18n.translate('xpack.infra.logs.alertFlyout.groupByOptimizationWarning', {
+              defaultMessage:
+                'When setting a group by we highly recommend using the {comparator} comparator for your threshold. This can lead to significant performance improvements.',
+              values: {
+                comparator: Comparator.GT,
+              },
+            })}
+          </EuiCallOut>
+        </>
+      )}
+
       <EuiSpacer size="l" />
     </>
   );

x-pack/plugins/infra/server/lib/alerting/log_threshold/log_threshold_chart_preview.ts

@@ -23,6 +23,7 @@ import {
   UngroupedSearchQueryResponse,
   GroupedSearchQueryResponse,
   GroupedSearchQueryResponseRT,
+  isOptimizedGroupedSearchQueryResponse,
 } from '../../../../common/alerting/logs/log_threshold/types';
 import { decodeOrThrow } from '../../../../common/runtime_types';
 import { ResolvedLogSourceConfiguration } from '../../../../common/log_sources';
@@ -97,10 +98,18 @@ const addHistogramAggregationToQuery = (
   };
 
   if (isGrouped) {
-    query.body.aggregations.groups.aggregations.filtered_results = {
-      ...query.body.aggregations.groups.aggregations.filtered_results,
-      aggregations: histogramAggregation,
-    };
+    // Optimized
+    if (!query.body.aggregations.groups.aggregations?.filtered_results) {
+      query.body.aggregations.groups.aggregations = {
+        ...query.body.aggregations.groups.aggregations,
+        ...histogramAggregation,
+      };
+    } else {
+      query.body.aggregations.groups.aggregations.filtered_results = {
+        ...query.body.aggregations.groups.aggregations.filtered_results,
+        aggregations: histogramAggregation,
+      };
+    }
   } else {
     query.body = {
       ...query.body,
@@ -151,18 +160,34 @@ const getGroupedResults = async (
 const processGroupedResults = (
   results: GroupedSearchQueryResponse['aggregations']['groups']['buckets']
 ): Series => {
-  return results.reduce<Series>((series, group) => {
-    if (!group.filtered_results.histogramBuckets) return series;
-    const groupName = Object.values(group.key).join(', ');
-    const points = group.filtered_results.histogramBuckets.buckets.reduce<Point[]>(
-      (pointsAcc, bucket) => {
+  const getGroupName = (
+    key: GroupedSearchQueryResponse['aggregations']['groups']['buckets'][0]['key']
+  ) => Object.values(key).join(', ');
+
+  if (isOptimizedGroupedSearchQueryResponse(results)) {
+    return results.reduce<Series>((series, group) => {
+      if (!group.histogramBuckets) return series;
+      const groupName = getGroupName(group.key);
+      const points = group.histogramBuckets.buckets.reduce<Point[]>((pointsAcc, bucket) => {
         const { key, doc_count: count } = bucket;
         return [...pointsAcc, { timestamp: key, value: count }];
-      },
-      []
-    );
-    return [...series, { id: groupName, points }];
-  }, []);
+      }, []);
+      return [...series, { id: groupName, points }];
+    }, []);
+  } else {
+    return results.reduce<Series>((series, group) => {
+      if (!group.filtered_results.histogramBuckets) return series;
+      const groupName = getGroupName(group.key);
+      const points = group.filtered_results.histogramBuckets.buckets.reduce<Point[]>(
+        (pointsAcc, bucket) => {
+          const { key, doc_count: count } = bucket;
+          return [...pointsAcc, { timestamp: key, value: count }];
+        },
+        []
+      );
+      return [...series, { id: groupName, points }];
+    }, []);
+  }
 };
 
 const processUngroupedResults = (results: UngroupedSearchQueryResponse): Series => {