Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[ResponseOps] Alert creation delay based on user definition #175851

Merged
merged 22 commits into from
Feb 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ const trackedAlertStateRt = t.type({
// count of consecutive recovered alerts for flapping
// will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
pendingRecoveredCount: t.number,
// count of consecutive active alerts will reset if the alert is recovered
activeCount: t.number,
});

export type TrackedLifecycleAlertState = t.TypeOf<typeof trackedAlertStateRt>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ export const metaSchema = schema.object({
// will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
pendingRecoveredCount: schema.maybe(schema.number()),
uuid: schema.maybe(schema.string()),
// count of consecutive active alerts
// will reset if the alert is recovered or if equal to notificationDelay.active stored in the rule
// count of consecutive active alerts will reset if the alert is recovered
activeCount: schema.maybe(schema.number()),
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

import { schema } from '@kbn/config-schema';
import { validateDurationV1, validateHoursV1, validateTimezoneV1 } from '../../../validation';
import { notifyWhenSchemaV1, notificationDelaySchemaV1 } from '../../../response';
import { notifyWhenSchemaV1, alertDelaySchemaV1 } from '../../../response';
import { alertsFilterQuerySchemaV1 } from '../../../../alerts_filter_query';

export const actionFrequencySchema = schema.object({
Expand Down Expand Up @@ -68,7 +68,7 @@ export const createBodySchema = schema.object({
}),
actions: schema.arrayOf(actionSchema, { defaultValue: [] }),
notify_when: schema.maybe(schema.nullable(notifyWhenSchemaV1)),
notification_delay: schema.maybe(notificationDelaySchemaV1),
alert_delay: schema.maybe(alertDelaySchemaV1),
});

export const createParamsSchema = schema.object({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ export interface CreateRuleRequestBody<Params extends RuleParamsV1 = never> {
schedule: CreateBodySchema['schedule'];
actions: CreateBodySchema['actions'];
notify_when?: CreateBodySchema['notify_when'];
notification_delay?: CreateBodySchema['notification_delay'];
alert_delay?: CreateBodySchema['alert_delay'];
}

export interface CreateRuleResponse<Params extends RuleParamsV1 = never> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export {
ruleSnoozeScheduleSchema as ruleSnoozeScheduleSchemaV1,
notifyWhenSchema as notifyWhenSchemaV1,
scheduleIdsSchema as scheduleIdsSchemaV1,
notificationDelaySchema as notificationDelaySchemaV1,
alertDelaySchema as alertDelaySchemaV1,
} from './schemas/v1';

export type {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ export const ruleSnoozeScheduleSchema = schema.object({
skipRecurrences: schema.maybe(schema.arrayOf(schema.string())),
});

export const notificationDelaySchema = schema.object({
export const alertDelaySchema = schema.object({
active: schema.number(),
});

Expand Down Expand Up @@ -218,7 +218,7 @@ export const ruleResponseSchema = schema.object({
revision: schema.number(),
running: schema.maybe(schema.nullable(schema.boolean())),
view_in_app_relative_url: schema.maybe(schema.nullable(schema.string())),
notification_delay: schema.maybe(notificationDelaySchema),
alert_delay: schema.maybe(alertDelaySchema),
});

export const scheduleIdsSchema = schema.maybe(schema.arrayOf(schema.string()));
Original file line number Diff line number Diff line change
Expand Up @@ -53,5 +53,5 @@ export interface RuleResponse<Params extends RuleParams = never> {
revision: RuleResponseSchemaType['revision'];
running?: RuleResponseSchemaType['running'];
view_in_app_relative_url?: RuleResponseSchemaType['view_in_app_relative_url'];
notification_delay?: RuleResponseSchemaType['notification_delay'];
alert_delay?: RuleResponseSchemaType['alert_delay'];
}
5 changes: 3 additions & 2 deletions x-pack/plugins/alerting/common/rule.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ export interface MappedParamsProperties {

export type MappedParams = SavedObjectAttributes & MappedParamsProperties;

export interface NotificationDelay {
export interface AlertDelay extends SavedObjectAttributes {
active: number;
}

Expand Down Expand Up @@ -178,7 +178,7 @@ export interface Rule<Params extends RuleTypeParams = never> {
revision: number;
running?: boolean | null;
viewInAppRelativeUrl?: string;
notificationDelay?: NotificationDelay;
alertDelay?: AlertDelay;
}

export interface SanitizedAlertsFilter extends AlertsFilter {
Expand Down Expand Up @@ -222,6 +222,7 @@ export type SanitizedRuleConfig = Pick<
| 'muteAll'
| 'revision'
| 'snoozeSchedule'
| 'alertDelay'
> & {
producer: string;
ruleTypeId: string;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,7 @@ describe('Alerts Client', () => {
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
notifyOnActionGroupChange: true,
maintenanceWindowIds: [],
alertDelay: 0,
};
});

Expand Down Expand Up @@ -516,6 +517,25 @@ describe('Alerts Client', () => {
});
});

test('should not index new alerts if the activeCount is less than the rule alertDelay', async () => {
const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>({
...alertsClientParams,
rule: { ...alertsClientParams.rule, alertDelay: 3 },
});

await alertsClient.initializeExecution(defaultExecutionOpts);

// Report 1 new alerts
const alertExecutorService = alertsClient.factory();
alertExecutorService.create('1').scheduleActions('default');

alertsClient.processAndLogAlerts(processAndLogAlertsOpts);

await alertsClient.persistAlerts();

expect(clusterClient.bulk).not.toHaveBeenCalled();
});

test('should update ongoing alerts in existing index', async () => {
clusterClient.search.mockResolvedValue({
took: 10,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -344,6 +344,11 @@ export class AlertsClient<
})
);
} else {
// skip writing the alert document if the number of consecutive
// active alerts is less than the rule alertDelay threshold
if (activeAlerts[id].getActiveCount() < this.options.rule.alertDelay) {
continue;
}
activeAlertsToIndex.push(
buildNewAlert<
AlertData,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ export const alertRuleData: AlertRuleData = {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
};

export const mockAAD = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ describe('Legacy Alerts Client', () => {
flappingSettings: DEFAULT_FLAPPING_SETTINGS,
notifyOnActionGroupChange: true,
maintenanceWindowIds: ['window-id1', 'window-id2'],
alertDelay: 5,
});

expect(processAlerts).toHaveBeenCalledWith({
Expand Down Expand Up @@ -275,13 +276,15 @@ describe('Legacy Alerts Client', () => {
},
true,
'default',
5,
{},
{
'1': new Alert<AlertInstanceContext, AlertInstanceContext>('1', testAlert1),
'2': new Alert<AlertInstanceContext, AlertInstanceContext>('2', testAlert2),
},
{},
{}
{},
null
);

expect(logAlerts).toHaveBeenCalledWith({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ export class LegacyAlertsClient<
notifyOnActionGroupChange,
flappingSettings,
maintenanceWindowIds,
alertDelay,
}: ProcessAlertsOpts) {
const {
newAlerts: processedAlertsNew,
Expand Down Expand Up @@ -168,10 +169,12 @@ export class LegacyAlertsClient<
flappingSettings,
notifyOnActionGroupChange,
this.options.ruleType.defaultActionGroupId,
alertDelay,
processedAlertsNew,
processedAlertsActive,
trimmedAlertsRecovered,
processedAlertsRecoveredCurrent
processedAlertsRecoveredCurrent,
this.startedAtString
);
alerts.currentRecoveredAlerts = merge(alerts.currentRecoveredAlerts, earlyRecoveredAlerts);

Expand Down Expand Up @@ -203,11 +206,13 @@ export class LegacyAlertsClient<
flappingSettings,
notifyOnActionGroupChange,
maintenanceWindowIds,
alertDelay,
}: ProcessAndLogAlertsOpts) {
this.processAlerts({
notifyOnActionGroupChange,
flappingSettings,
maintenanceWindowIds,
alertDelay,
});

this.logAlerts({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ describe('formatRule', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
ruleType,
})
Expand Down
3 changes: 3 additions & 0 deletions x-pack/plugins/alerting/server/alerts_client/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ export interface AlertRuleData {
revision: number;
spaceId: string;
tags: string[];
alertDelay: number;
}

export interface AlertRule {
Expand Down Expand Up @@ -111,12 +112,14 @@ export interface ProcessAndLogAlertsOpts {
flappingSettings: RulesSettingsFlappingProperties;
notifyOnActionGroupChange: boolean;
maintenanceWindowIds: string[];
alertDelay: number;
}

export interface ProcessAlertsOpts {
flappingSettings: RulesSettingsFlappingProperties;
notifyOnActionGroupChange: boolean;
maintenanceWindowIds: string[];
alertDelay: number;
}

export interface LogAlertsOpts {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1475,6 +1475,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});

Expand All @@ -1495,6 +1496,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
kibanaVersion: '8.8.0',
});
Expand Down Expand Up @@ -1528,6 +1530,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});

Expand Down Expand Up @@ -1576,6 +1579,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});

Expand Down Expand Up @@ -1610,6 +1614,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
kibanaVersion: '8.8.0',
});
Expand Down Expand Up @@ -1674,6 +1679,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
}),
alertsService.createAlertsClient({
Expand All @@ -1691,6 +1697,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
}),
]);
Expand Down Expand Up @@ -1725,6 +1732,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
kibanaVersion: '8.8.0',
});
Expand Down Expand Up @@ -1781,6 +1789,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});

Expand All @@ -1801,6 +1810,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
kibanaVersion: '8.8.0',
});
Expand Down Expand Up @@ -1865,6 +1875,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});
};
Expand Down Expand Up @@ -1892,6 +1903,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
kibanaVersion: '8.8.0',
});
Expand Down Expand Up @@ -1961,6 +1973,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});
};
Expand Down Expand Up @@ -2026,6 +2039,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});

Expand Down Expand Up @@ -2091,6 +2105,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});

Expand Down Expand Up @@ -2154,6 +2169,7 @@ describe('Alerts Service', () => {
revision: 0,
spaceId: 'default',
tags: ['rule-', '-tags'],
alertDelay: 0,
},
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@

import { schema } from '@kbn/config-schema';
import { validateDuration } from '../../../validation';
import {
notifyWhenSchema,
actionAlertsFilterSchema,
notificationDelaySchema,
} from '../../../schemas';
import { notifyWhenSchema, actionAlertsFilterSchema, alertDelaySchema } from '../../../schemas';

export const createRuleDataSchema = schema.object({
name: schema.string(),
Expand Down Expand Up @@ -44,5 +40,5 @@ export const createRuleDataSchema = schema.object({
{ defaultValue: [] }
),
notifyWhen: schema.maybe(schema.nullable(notifyWhenSchema)),
notificationDelay: schema.maybe(notificationDelaySchema),
alertDelay: schema.maybe(alertDelaySchema),
});
Loading