Skip to content

Commit

Permalink
[Logs UI] Log threshold rule performance improvements (elastic#102650)
Browse files Browse the repository at this point in the history 
* Add optimisations for executor / chart previews

Co-authored-by: Felix Stürmer <weltenwort@users.noreply.github.com>
  • Loading branch information
@Kerry350 @weltenwort
Kerry350 and weltenwort committed Jun 24, 2021
1 parent 67d4c31 commit 9ba1ead
Show file tree
Hide file tree
Showing 7 changed files with 483 additions and 335 deletions.
108 changes: 80 additions & 28 deletions x-pack/plugins/infra/common/alerting/logs/log_threshold/types.ts
View file
Expand Up @@ -100,7 +100,7 @@ export enum AlertStates {
ERROR,
}

const ThresholdRT = rt.type({
export const ThresholdRT = rt.type({
comparator: ComparatorRT,
value: rt.number,
});
Expand Down Expand Up @@ -240,65 +240,117 @@ const chartPreviewHistogramBucket = rt.type({
doc_count: rt.number,
});

const ChartPreviewBucketsRT = rt.partial({
histogramBuckets: rt.type({
buckets: rt.array(chartPreviewHistogramBucket),
}),
});

// ES query responses //
const hitsRT = rt.type({
total: rt.type({
value: rt.number,
}),
});

const bucketFieldsRT = rt.type({
key: rt.record(rt.string, rt.string),
doc_count: rt.number,
});

const afterKeyRT = rt.partial({
after_key: rt.record(rt.string, rt.string),
});

export const UngroupedSearchQueryResponseRT = rt.intersection([
commonSearchSuccessResponseFieldsRT,
rt.intersection([
rt.type({
hits: rt.type({
total: rt.type({
value: rt.number,
}),
}),
hits: hitsRT,
}),
// Chart preview buckets
rt.partial({
aggregations: rt.type({
histogramBuckets: rt.type({
buckets: rt.array(chartPreviewHistogramBucket),
}),
}),
aggregations: ChartPreviewBucketsRT,
}),
]),
]);

export type UngroupedSearchQueryResponse = rt.TypeOf<typeof UngroupedSearchQueryResponseRT>;

export const GroupedSearchQueryResponseRT = rt.intersection([
export const UnoptimizedGroupedSearchQueryResponseRT = rt.intersection([
commonSearchSuccessResponseFieldsRT,
rt.type({
aggregations: rt.type({
groups: rt.intersection([
rt.type({
buckets: rt.array(
rt.type({
key: rt.record(rt.string, rt.string),
doc_count: rt.number,
...bucketFieldsRT.props,
filtered_results: rt.intersection([
rt.type({
doc_count: rt.number,
}),
// Chart preview buckets
rt.partial({
histogramBuckets: rt.type({
buckets: rt.array(chartPreviewHistogramBucket),
}),
}),
ChartPreviewBucketsRT,
]),
})
),
}),
rt.partial({
after_key: rt.record(rt.string, rt.string),
}),
afterKeyRT,
]),
}),
hits: rt.type({
total: rt.type({
value: rt.number,
}),
hits: hitsRT,
}),
]);

export type UnoptimizedGroupedSearchQueryResponse = rt.TypeOf<
typeof UnoptimizedGroupedSearchQueryResponseRT
>;

export const OptimizedGroupedSearchQueryResponseRT = rt.intersection([
commonSearchSuccessResponseFieldsRT,
rt.type({
aggregations: rt.type({
groups: rt.intersection([
rt.type({
buckets: rt.array(rt.intersection([bucketFieldsRT, ChartPreviewBucketsRT])),
}),
afterKeyRT,
]),
}),
hits: hitsRT,
}),
]);

export type OptimizedGroupedSearchQueryResponse = rt.TypeOf<
typeof OptimizedGroupedSearchQueryResponseRT
>;

export const GroupedSearchQueryResponseRT = rt.union([
UnoptimizedGroupedSearchQueryResponseRT,
OptimizedGroupedSearchQueryResponseRT,
]);

export type GroupedSearchQueryResponse = rt.TypeOf<typeof GroupedSearchQueryResponseRT>;

export const isOptimizedGroupedSearchQueryResponse = (
response: GroupedSearchQueryResponse['aggregations']['groups']['buckets']
): response is OptimizedGroupedSearchQueryResponse['aggregations']['groups']['buckets'] => {
const result = response[0];
return result && !result.hasOwnProperty('filtered_results');
};

export const isOptimizableGroupedThreshold = (
selectedComparator: AlertParams['count']['comparator'],
selectedValue?: AlertParams['count']['value']
) => {
if (selectedComparator === Comparator.GT) {
return true;
} else if (
typeof selectedValue === 'number' &&
selectedComparator === Comparator.GT_OR_EQ &&
selectedValue > 0
) {
return true;
} else {
return false;
}
};
9 changes: 9 additions & 0 deletions x-pack/plugins/infra/common/http_api/log_alerts/chart_preview_data.ts
View file
Expand Up @@ -7,6 +7,7 @@

import * as rt from 'io-ts';
import {
ThresholdRT,
countCriteriaRT,
timeUnitRT,
timeSizeRT,
Expand Down Expand Up @@ -58,6 +59,14 @@ export type GetLogAlertsChartPreviewDataSuccessResponsePayload = rt.TypeOf<
export const getLogAlertsChartPreviewDataAlertParamsSubsetRT: any = rt.intersection([
rt.type({
criteria: countCriteriaRT,
count: rt.intersection([
rt.type({
comparator: ThresholdRT.props.comparator,
}),
rt.partial({
value: ThresholdRT.props.value,
}),
]),
timeUnit: timeUnitRT,
timeSize: timeSizeRT,
}),
Expand Down
13 changes: 12 additions & 1 deletion ...ra/public/alerting/log_threshold/components/expression_editor/criterion_preview_chart.tsx
View file
Expand Up @@ -68,6 +68,10 @@ export const CriterionPreview: React.FC<Props> = ({
const criteria = field && comparator && value ? [{ field, comparator, value }] : [];
const params = {
criteria,
count: {
comparator: alertParams.count.comparator,
value: alertParams.count.value,
},
timeSize: alertParams.timeSize,
timeUnit: alertParams.timeUnit,
groupBy: alertParams.groupBy,
Expand All @@ -78,7 +82,14 @@ export const CriterionPreview: React.FC<Props> = ({
} catch (error) {
return null;
}
}, [alertParams.timeSize, alertParams.timeUnit, alertParams.groupBy, chartCriterion]);
}, [
alertParams.timeSize,
alertParams.timeUnit,
alertParams.groupBy,
alertParams.count.comparator,
alertParams.count.value,
chartCriterion,
]);

// Check for the existence of properties that are necessary for a meaningful chart.
if (chartAlertParams === null || chartAlertParams.criteria.length === 0) return null;
Expand Down
25 changes: 25 additions & 0 deletions x-pack/plugins/infra/public/alerting/log_threshold/components/expression_editor/editor.tsx
View file
Expand Up @@ -23,6 +23,7 @@ import {
PartialRatioAlertParams,
ThresholdType,
timeUnitRT,
isOptimizableGroupedThreshold,
} from '../../../../../common/alerting/logs/log_threshold/types';
import { decodeOrThrow } from '../../../../../common/runtime_types';
import { ObjectEntries } from '../../../../../common/utility_types';
Expand Down Expand Up @@ -255,6 +256,15 @@ export const Editor: React.FC<
setHasSetDefaults(true);
});

const shouldShowGroupByOptimizationWarning = useMemo(() => {
const hasSetGroupBy = alertParams.groupBy && alertParams.groupBy.length > 0;
return (
hasSetGroupBy &&
alertParams.count &&
!isOptimizableGroupedThreshold(alertParams.count.comparator, alertParams.count.value)
);
}, [alertParams]);

// Wait until the alert param defaults have been set
if (!hasSetDefaults) return null;

Expand Down Expand Up @@ -299,6 +309,21 @@ export const Editor: React.FC<

{alertParams.criteria && isRatioAlert(alertParams.criteria) && criteriaComponent}

{shouldShowGroupByOptimizationWarning && (
<>
<EuiSpacer size="l" />
<EuiCallOut color="warning">
{i18n.translate('xpack.infra.logs.alertFlyout.groupByOptimizationWarning', {
defaultMessage:
'When setting a "group by" we highly recommend using the "{comparator}" comparator for your threshold. This can lead to significant performance improvements.',
values: {
comparator: Comparator.GT,
},
})}
</EuiCallOut>
</>
)}

<EuiSpacer size="l" />
</>
);
Expand Down
54 changes: 40 additions & 14 deletions x-pack/plugins/infra/server/lib/alerting/log_threshold/log_threshold_chart_preview.ts
View file
Expand Up @@ -23,6 +23,7 @@ import {
UngroupedSearchQueryResponse,
GroupedSearchQueryResponse,
GroupedSearchQueryResponseRT,
isOptimizedGroupedSearchQueryResponse,
} from '../../../../common/alerting/logs/log_threshold/types';
import { decodeOrThrow } from '../../../../common/runtime_types';
import { ResolvedLogSourceConfiguration } from '../../../../common/log_sources';
Expand Down Expand Up @@ -97,10 +98,19 @@ const addHistogramAggregationToQuery = (
};

if (isGrouped) {
query.body.aggregations.groups.aggregations.filtered_results = {
...query.body.aggregations.groups.aggregations.filtered_results,
aggregations: histogramAggregation,
};
const isOptimizedQuery = !query.body.aggregations.groups.aggregations?.filtered_results;

if (isOptimizedQuery) {
query.body.aggregations.groups.aggregations = {
...query.body.aggregations.groups.aggregations,
...histogramAggregation,
};
} else {
query.body.aggregations.groups.aggregations.filtered_results = {
...query.body.aggregations.groups.aggregations.filtered_results,
aggregations: histogramAggregation,
};
}
} else {
query.body = {
...query.body,
Expand Down Expand Up @@ -151,18 +161,34 @@ const getGroupedResults = async (
const processGroupedResults = (
results: GroupedSearchQueryResponse['aggregations']['groups']['buckets']
): Series => {
return results.reduce<Series>((series, group) => {
if (!group.filtered_results.histogramBuckets) return series;
const groupName = Object.values(group.key).join(', ');
const points = group.filtered_results.histogramBuckets.buckets.reduce<Point[]>(
(pointsAcc, bucket) => {
const getGroupName = (
key: GroupedSearchQueryResponse['aggregations']['groups']['buckets'][0]['key']
) => Object.values(key).join(', ');

if (isOptimizedGroupedSearchQueryResponse(results)) {
return results.reduce<Series>((series, group) => {
if (!group.histogramBuckets) return series;
const groupName = getGroupName(group.key);
const points = group.histogramBuckets.buckets.reduce<Point[]>((pointsAcc, bucket) => {
const { key, doc_count: count } = bucket;
return [...pointsAcc, { timestamp: key, value: count }];
},
[]
);
return [...series, { id: groupName, points }];
}, []);
}, []);
return [...series, { id: groupName, points }];
}, []);
} else {
return results.reduce<Series>((series, group) => {
if (!group.filtered_results.histogramBuckets) return series;
const groupName = getGroupName(group.key);
const points = group.filtered_results.histogramBuckets.buckets.reduce<Point[]>(
(pointsAcc, bucket) => {
const { key, doc_count: count } = bucket;
return [...pointsAcc, { timestamp: key, value: count }];
},
[]
);
return [...series, { id: groupName, points }];
}, []);
}
};

const processUngroupedResults = (results: UngroupedSearchQueryResponse): Series => {
Expand Down

0 comments on commit 9ba1ead

Please sign in to comment.