diff --git a/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts index 8fa5cb111f6d91..f82791dd4e4ea1 100644 --- a/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts +++ b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts @@ -20,6 +20,8 @@ const trackedAlertStateRt = t.type({ // count of consecutive recovered alerts for flapping // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings pendingRecoveredCount: t.number, + // count of consecutive active alerts will reset if the alert is recovered + activeCount: t.number, }); export type TrackedLifecycleAlertState = t.TypeOf; diff --git a/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts b/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts index 91353fc3cd1202..247cc7124c000b 100644 --- a/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts +++ b/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts @@ -36,8 +36,7 @@ export const metaSchema = schema.object({ // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings pendingRecoveredCount: schema.maybe(schema.number()), uuid: schema.maybe(schema.string()), - // count of consecutive active alerts - // will reset if the alert is recovered or if equal to alertDelay.active stored in the rule + // count of consecutive active alerts will reset if the alert is recovered activeCount: schema.maybe(schema.number()), }); diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index db888302b5d00b..7cec5bdbdd7a62 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -222,6 +222,7 @@ export type SanitizedRuleConfig = Pick< | 'muteAll' | 'revision' | 'snoozeSchedule' + | 'alertDelay' > & { producer: string; ruleTypeId: string; diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts index d00e1dfafb0830..e587bb311137d0 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts @@ -308,6 +308,7 @@ describe('Alerts Client', () => { flappingSettings: DEFAULT_FLAPPING_SETTINGS, notifyOnActionGroupChange: true, maintenanceWindowIds: [], + alertDelay: 0, }; }); @@ -516,6 +517,25 @@ describe('Alerts Client', () => { }); }); + test('should not index new alerts if the activeCount is less than the rule alertDelay', async () => { + const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>({ + ...alertsClientParams, + rule: { ...alertsClientParams.rule, alertDelay: 3 }, + }); + + await alertsClient.initializeExecution(defaultExecutionOpts); + + // Report 1 new alerts + const alertExecutorService = alertsClient.factory(); + alertExecutorService.create('1').scheduleActions('default'); + + alertsClient.processAndLogAlerts(processAndLogAlertsOpts); + + await alertsClient.persistAlerts(); + + expect(clusterClient.bulk).not.toHaveBeenCalled(); + }); + test('should update ongoing alerts in existing index', async () => { clusterClient.search.mockResolvedValue({ took: 10, diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts index 144a0203e4909f..30c9ee669f0c7f 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts @@ -343,6 +343,11 @@ export class AlertsClient< }) ); } else { + // skip writing the alert document if the number of consecutive + // active alerts is less than the rule alertDelay threshold + if (activeAlerts[id].getActiveCount() < this.options.rule.alertDelay) { + continue; + } activeAlertsToIndex.push( buildNewAlert< AlertData, diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts index 90c2f4c40b65bd..0da20a5e49b70f 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts @@ -25,6 +25,7 @@ export const alertRuleData: AlertRuleData = { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }; export const mockAAD = { diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts index 446ecfd79c8b80..d01060820ae524 100644 --- a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts @@ -245,6 +245,7 @@ describe('Legacy Alerts Client', () => { flappingSettings: DEFAULT_FLAPPING_SETTINGS, notifyOnActionGroupChange: true, maintenanceWindowIds: ['window-id1', 'window-id2'], + alertDelay: 5, }); expect(processAlerts).toHaveBeenCalledWith({ @@ -275,13 +276,15 @@ describe('Legacy Alerts Client', () => { }, true, 'default', + 5, {}, { '1': new Alert('1', testAlert1), '2': new Alert('2', testAlert2), }, {}, - {} + {}, + null ); expect(logAlerts).toHaveBeenCalledWith({ diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts index 0de97a8a29be68..a5cfc642a19ee8 100644 --- a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts +++ b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts @@ -140,6 +140,7 @@ export class LegacyAlertsClient< notifyOnActionGroupChange, flappingSettings, maintenanceWindowIds, + alertDelay, }: ProcessAlertsOpts) { const { newAlerts: processedAlertsNew, @@ -168,10 +169,12 @@ export class LegacyAlertsClient< flappingSettings, notifyOnActionGroupChange, this.options.ruleType.defaultActionGroupId, + alertDelay, processedAlertsNew, processedAlertsActive, trimmedAlertsRecovered, - processedAlertsRecoveredCurrent + processedAlertsRecoveredCurrent, + this.startedAtString ); alerts.currentRecoveredAlerts = merge(alerts.currentRecoveredAlerts, earlyRecoveredAlerts); @@ -203,11 +206,13 @@ export class LegacyAlertsClient< flappingSettings, notifyOnActionGroupChange, maintenanceWindowIds, + alertDelay, }: ProcessAndLogAlertsOpts) { this.processAlerts({ notifyOnActionGroupChange, flappingSettings, maintenanceWindowIds, + alertDelay, }); this.logAlerts({ diff --git a/x-pack/plugins/alerting/server/alerts_client/types.ts b/x-pack/plugins/alerting/server/alerts_client/types.ts index c5d02f1cd6f141..c38351126af970 100644 --- a/x-pack/plugins/alerting/server/alerts_client/types.ts +++ b/x-pack/plugins/alerting/server/alerts_client/types.ts @@ -47,6 +47,7 @@ export interface AlertRuleData { revision: number; spaceId: string; tags: string[]; + alertDelay: number; } export interface AlertRule { @@ -111,12 +112,14 @@ export interface ProcessAndLogAlertsOpts { flappingSettings: RulesSettingsFlappingProperties; notifyOnActionGroupChange: boolean; maintenanceWindowIds: string[]; + alertDelay: number; } export interface ProcessAlertsOpts { flappingSettings: RulesSettingsFlappingProperties; notifyOnActionGroupChange: boolean; maintenanceWindowIds: string[]; + alertDelay: number; } export interface LogAlertsOpts { diff --git a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts index c1465d5b7a2388..4656f4377f1301 100644 --- a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts @@ -21,6 +21,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, { '1': alert1, }, @@ -89,6 +90,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, {}, {}, { @@ -222,6 +224,7 @@ describe('getAlertsForNotification', () => { DISABLE_FLAPPING_SETTINGS, true, 'default', + 0, {}, {}, { @@ -353,6 +356,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, false, 'default', + 0, {}, {}, { @@ -455,10 +459,11 @@ describe('getAlertsForNotification', () => { }); const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } }); - const { newAlerts, activeAlerts } = getAlertsForNotification( + const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification( DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, { '1': alert1, }, @@ -507,6 +512,30 @@ describe('getAlertsForNotification', () => { }, } `); + expect(currentActiveAlerts).toMatchInlineSnapshot(` + Object { + "1": Object { + "meta": Object { + "activeCount": 2, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-1", + }, + "state": Object {}, + }, + "2": Object { + "meta": Object { + "activeCount": 1, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-2", + }, + "state": Object {}, + }, + } + `); }); test('should reset activeCount for all recovered alerts', () => { @@ -517,6 +546,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, {}, {}, { @@ -574,4 +604,78 @@ describe('getAlertsForNotification', () => { } `); }); + + test('should remove the alert from newAlerts and should not return the alert in currentActiveAlerts if the activeCount is less than the rule alertDelay', () => { + const alert1 = new Alert('1', { + meta: { activeCount: 1, uuid: 'uuid-1' }, + }); + const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } }); + + const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification( + DEFAULT_FLAPPING_SETTINGS, + true, + 'default', + 5, + { + '1': alert1, + }, + { + '1': alert1, + '2': alert2, + }, + {}, + {} + ); + expect(newAlerts).toMatchInlineSnapshot(`Object {}`); + expect(activeAlerts).toMatchInlineSnapshot(` + Object { + "1": Object { + "meta": Object { + "activeCount": 2, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-1", + }, + "state": Object {}, + }, + "2": Object { + "meta": Object { + "activeCount": 1, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-2", + }, + "state": Object {}, + }, + } + `); + expect(currentActiveAlerts).toMatchInlineSnapshot(`Object {}`); + }); + + test('should update active alert to look like a new alert if the activeCount is equal to the rule alertDelay', () => { + const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } }); + + const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification( + DEFAULT_FLAPPING_SETTINGS, + true, + 'default', + 1, + {}, + { + '2': alert2, + }, + {}, + {} + ); + expect(newAlerts['2'].getState().duration).toBe('0'); + expect(newAlerts['2'].getState().start).toBeTruthy(); + + expect(activeAlerts['2'].getState().duration).toBe('0'); + expect(activeAlerts['2'].getState().start).toBeTruthy(); + + expect(currentActiveAlerts['2'].getState().duration).toBe('0'); + expect(currentActiveAlerts['2'].getState().start).toBeTruthy(); + }); }); diff --git a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts index 63e95402549a4a..593d92b35383ba 100644 --- a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts +++ b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts @@ -19,10 +19,12 @@ export function getAlertsForNotification< flappingSettings: RulesSettingsFlappingProperties, notifyOnActionGroupChange: boolean, actionGroupId: string, + alertDelay: number, newAlerts: Record> = {}, activeAlerts: Record> = {}, recoveredAlerts: Record> = {}, - currentRecoveredAlerts: Record> = {} + currentRecoveredAlerts: Record> = {}, + startedAt?: string | null ) { const currentActiveAlerts: Record> = {}; @@ -30,7 +32,22 @@ export function getAlertsForNotification< const alert = activeAlerts[id]; alert.incrementActiveCount(); alert.resetPendingRecoveredCount(); - currentActiveAlerts[id] = alert; + // do not trigger an action notification if the number of consecutive + // active alerts is less than the rule alertDelay threshold + if (alert.getActiveCount() < alertDelay) { + // remove from new alerts + delete newAlerts[id]; + } else { + currentActiveAlerts[id] = alert; + // if the active count is equal to the alertDelay it is considered a new alert + if (alert.getActiveCount() === alertDelay) { + const currentTime = startedAt ?? new Date().toISOString(); + const state = alert.getState(); + // keep the state and update the start time and duration + alert.replaceState({ ...state, start: currentTime, duration: '0' }); + newAlerts[id] = alert; + } + } } for (const id of keys(currentRecoveredAlerts)) { diff --git a/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts b/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts index 9fe8f1a8202771..a742afb152b198 100644 --- a/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts @@ -2052,165 +2052,6 @@ describe('Execution Handler', () => { `); }); - test('does not schedule actions for alerts with activeCount less than the alertDelay.active threshold', async () => { - const executionHandler = new ExecutionHandler( - generateExecutionParams({ - ...defaultExecutionParams, - rule: { - ...defaultExecutionParams.rule, - alertDelay: { - active: 3, - }, - }, - }) - ); - - await executionHandler.run({ - ...generateAlert({ id: 1 }), - ...generateAlert({ id: 2, activeCount: 2 }), - }); - - expect(actionsClient.bulkEnqueueExecution).not.toHaveBeenCalled(); - expect(defaultExecutionParams.logger.debug).toHaveBeenCalledTimes(2); - - expect(defaultExecutionParams.logger.debug).toHaveBeenCalledWith( - 'no scheduling of action "1" for rule "1": the alert activeCount: 0 is less than the rule alertDelay.active: 3 threshold.' - ); - expect(defaultExecutionParams.logger.debug).toHaveBeenCalledWith( - 'no scheduling of action "1" for rule "1": the alert activeCount: 2 is less than the rule alertDelay.active: 3 threshold.' - ); - }); - - test('schedules actions for alerts with activeCount greater than or equal the alertDelay.active threshold', async () => { - const executionHandler = new ExecutionHandler( - generateExecutionParams({ - ...defaultExecutionParams, - rule: { - ...defaultExecutionParams.rule, - alertDelay: { - active: 3, - }, - }, - }) - ); - - await executionHandler.run({ - ...generateAlert({ id: 1, activeCount: 3 }), - ...generateAlert({ id: 2, activeCount: 4 }), - }); - - expect(actionsClient.bulkEnqueueExecution).toHaveBeenCalledTimes(1); - expect(actionsClient.bulkEnqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "actionTypeId": "test", - "apiKey": "MTIzOmFiYw==", - "consumer": "rule-consumer", - "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28", - "id": "1", - "params": Object { - "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 1 goes here", - "contextVal": "My goes here", - "foo": true, - "stateVal": "My goes here", - }, - "relatedSavedObjects": Array [ - Object { - "id": "1", - "namespace": "test1", - "type": "alert", - "typeId": "test", - }, - ], - "source": Object { - "source": Object { - "id": "1", - "type": "alert", - }, - "type": "SAVED_OBJECT", - }, - "spaceId": "test1", - }, - Object { - "actionTypeId": "test", - "apiKey": "MTIzOmFiYw==", - "consumer": "rule-consumer", - "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28", - "id": "1", - "params": Object { - "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 2 goes here", - "contextVal": "My goes here", - "foo": true, - "stateVal": "My goes here", - }, - "relatedSavedObjects": Array [ - Object { - "id": "1", - "namespace": "test1", - "type": "alert", - "typeId": "test", - }, - ], - "source": Object { - "source": Object { - "id": "1", - "type": "alert", - }, - "type": "SAVED_OBJECT", - }, - "spaceId": "test1", - }, - ], - ] - `); - }); - - test('schedules actions if alertDelay.active threshold is not defined', async () => { - const executionHandler = new ExecutionHandler(generateExecutionParams()); - - await executionHandler.run({ - ...generateAlert({ id: 1, activeCount: 1 }), - }); - - expect(actionsClient.bulkEnqueueExecution).toHaveBeenCalledTimes(1); - expect(actionsClient.bulkEnqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "actionTypeId": "test", - "apiKey": "MTIzOmFiYw==", - "consumer": "rule-consumer", - "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28", - "id": "1", - "params": Object { - "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 1 goes here", - "contextVal": "My goes here", - "foo": true, - "stateVal": "My goes here", - }, - "relatedSavedObjects": Array [ - Object { - "id": "1", - "namespace": "test1", - "type": "alert", - "typeId": "test", - }, - ], - "source": Object { - "source": Object { - "id": "1", - "type": "alert", - }, - "type": "SAVED_OBJECT", - }, - "spaceId": "test1", - }, - ], - ] - `); - }); - describe('rule url', () => { const ruleWithUrl = { ...rule, diff --git a/x-pack/plugins/alerting/server/task_runner/execution_handler.ts b/x-pack/plugins/alerting/server/task_runner/execution_handler.ts index 10a4d7484b0c6a..ec690bb8ba0f5f 100644 --- a/x-pack/plugins/alerting/server/task_runner/execution_handler.ts +++ b/x-pack/plugins/alerting/server/task_runner/execution_handler.ts @@ -628,19 +628,6 @@ export class ExecutionHandler< continue; } - if (this.rule.alertDelay && alert.getActiveCount() < this.rule.alertDelay.active) { - this.logger.debug( - `no scheduling of action "${action.id}" for rule "${ - this.taskInstance.params.alertId - }": the alert activeCount: ${alert.getActiveCount()} is less than the rule alertDelay.active: ${ - this.rule.alertDelay.active - } threshold.` - ); - continue; - } else { - alert.resetActiveCount(); - } - const actionGroup = this.getActionGroup(alert); if (!this.ruleTypeActionGroups!.has(actionGroup)) { diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index 3d647966414f5c..b2a984ea5768fa 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -454,7 +454,7 @@ export const generateAlertInstance = ( flapping: false, maintenanceWindowIds: maintenanceWindowIds || [], pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { bar: false, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index d14d44010252d8..e4afa351d4f146 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -2954,7 +2954,7 @@ describe('Task Runner', () => { maintenanceWindowIds: [], flapping: false, pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { duration: '0', @@ -3125,7 +3125,7 @@ describe('Task Runner', () => { maintenanceWindowIds: [], flapping: false, pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { duration: '0', @@ -3143,7 +3143,7 @@ describe('Task Runner', () => { maintenanceWindowIds: [], flapping: false, pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { duration: '0', diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 208e46b88a1f66..f3c0f39b1e0a1a 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -257,6 +257,7 @@ export class TaskRunner< revision: rule.revision, spaceId, tags: rule.tags, + alertDelay: rule.alertDelay?.active ?? 0, }; } @@ -582,6 +583,7 @@ export class TaskRunner< notifyWhen === RuleNotifyWhen.CHANGE || some(actions, (action) => action.frequency?.notifyWhen === RuleNotifyWhen.CHANGE), maintenanceWindowIds: maintenanceWindowsWithoutScopedQueryIds, + alertDelay: rule.alertDelay?.active ?? 0, }); }); diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts index afed418ec8d3da..2367167b496971 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts @@ -208,6 +208,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -216,6 +217,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -358,6 +360,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -366,6 +369,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -490,6 +494,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -498,6 +503,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -732,6 +738,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -740,6 +747,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -850,6 +858,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -858,6 +867,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlerts: {}, @@ -963,6 +973,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -971,6 +982,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1077,6 +1089,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: { @@ -1087,6 +1100,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, }, @@ -1258,6 +1272,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1266,6 +1281,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1388,6 +1404,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1396,6 +1413,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1570,6 +1588,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: flapping, flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1578,6 +1597,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [false, false], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_2: { alertId: 'TEST_ALERT_2', @@ -1586,6 +1606,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: flapping, flapping: true, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_3: { alertId: 'TEST_ALERT_3', @@ -1594,6 +1615,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [false, false], flapping: true, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1604,6 +1626,7 @@ describe('createLifecycleExecutor', () => { expect(serializedAlerts.state.trackedAlerts).toEqual({ TEST_ALERT_0: { + activeCount: 1, alertId: 'TEST_ALERT_0', alertUuid: 'TEST_ALERT_0_UUID', flapping: true, @@ -1612,6 +1635,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-01T12:00:00.000Z', }, TEST_ALERT_1: { + activeCount: 1, alertId: 'TEST_ALERT_1', alertUuid: 'TEST_ALERT_1_UUID', flapping: false, @@ -1620,6 +1644,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-02T12:00:00.000Z', }, TEST_ALERT_2: { + activeCount: 1, alertId: 'TEST_ALERT_2', alertUuid: 'TEST_ALERT_2_UUID', flapping: true, @@ -1628,6 +1653,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-01T12:00:00.000Z', }, TEST_ALERT_3: { + activeCount: 1, alertId: 'TEST_ALERT_3', alertUuid: 'TEST_ALERT_3_UUID', flapping: true, @@ -1786,6 +1812,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [true, true, true, true], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1794,6 +1821,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: notFlapping, flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_2: { alertId: 'TEST_ALERT_2', @@ -1802,6 +1830,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [true, true], flapping: true, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_3: { alertId: 'TEST_ALERT_3', @@ -1810,6 +1839,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: notFlapping, flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1820,6 +1850,7 @@ describe('createLifecycleExecutor', () => { expect(serializedAlerts.state.trackedAlerts).toEqual({ TEST_ALERT_2: { + activeCount: 0, alertId: 'TEST_ALERT_2', alertUuid: 'TEST_ALERT_2_UUID', flapping: true, @@ -1831,6 +1862,7 @@ describe('createLifecycleExecutor', () => { expect(serializedAlerts.state.trackedAlertsRecovered).toEqual({ TEST_ALERT_0: { + activeCount: 0, alertId: 'TEST_ALERT_0', alertUuid: 'TEST_ALERT_0_UUID', flapping: true, @@ -1839,6 +1871,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-01T12:00:00.000Z', }, TEST_ALERT_1: { + activeCount: 0, alertId: 'TEST_ALERT_1', alertUuid: 'TEST_ALERT_1_UUID', flapping: false, @@ -1847,6 +1880,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-02T12:00:00.000Z', }, TEST_ALERT_3: { + activeCount: 0, alertId: 'TEST_ALERT_3', alertUuid: 'TEST_ALERT_3_UUID', flapping: false, diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 91d30fae7b3dc9..994187484f9719 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -290,7 +290,7 @@ export const createLifecycleExecutor = trackedAlertRecoveredIds ); - const { alertUuid, started, flapping, pendingRecoveredCount } = !isNew + const { alertUuid, started, flapping, pendingRecoveredCount, activeCount } = !isNew ? state.trackedAlerts[alertId] : { alertUuid: lifecycleAlertServices.getAlertUuid(alertId), @@ -299,6 +299,7 @@ export const createLifecycleExecutor = ? state.trackedAlertsRecovered[alertId].flapping : false, pendingRecoveredCount: 0, + activeCount: 0, }; const event: ParsedTechnicalFields & ParsedExperimentalFields = { @@ -342,16 +343,21 @@ export const createLifecycleExecutor = flappingHistory, flapping, pendingRecoveredCount, + activeCount, }; }); const trackedEventsToIndex = makeEventsDataMapFor(trackedAlertIds); const newEventsToIndex = makeEventsDataMapFor(newAlertIds); const trackedRecoveredEventsToIndex = makeEventsDataMapFor(trackedAlertRecoveredIds); - const allEventsToIndex = [ - ...getAlertsForNotification(flappingSettings, trackedEventsToIndex), - ...newEventsToIndex, - ]; + const allEventsToIndex = getAlertsForNotification( + commonRuleFields[TIMESTAMP], + flappingSettings, + rule.alertDelay?.active ?? 0, + trackedEventsToIndex, + newEventsToIndex, + maintenanceWindowIds + ); // Only write alerts if: // - writing is enabled @@ -392,18 +398,34 @@ export const createLifecycleExecutor = } const nextTrackedAlerts = Object.fromEntries( - allEventsToIndex + [...allEventsToIndex, ...trackedEventsToIndex] .filter(({ event }) => event[ALERT_STATUS] !== ALERT_STATUS_RECOVERED) - .map(({ event, flappingHistory, flapping: isCurrentlyFlapping, pendingRecoveredCount }) => { - const alertId = event[ALERT_INSTANCE_ID]!; - const alertUuid = event[ALERT_UUID]!; - const started = new Date(event[ALERT_START]!).toISOString(); - const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); - return [ - alertId, - { alertId, alertUuid, started, flappingHistory, flapping, pendingRecoveredCount }, - ]; - }) + .map( + ({ + event, + flappingHistory, + flapping: isCurrentlyFlapping, + pendingRecoveredCount, + activeCount, + }) => { + const alertId = event[ALERT_INSTANCE_ID]!; + const alertUuid = event[ALERT_UUID]!; + const started = new Date(event[ALERT_START]!).toISOString(); + const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); + return [ + alertId, + { + alertId, + alertUuid, + started, + flappingHistory, + flapping, + pendingRecoveredCount, + activeCount, + }, + ]; + } + ) ); const nextTrackedAlertsRecovered = Object.fromEntries( @@ -416,16 +438,32 @@ export const createLifecycleExecutor = event[ALERT_STATUS] === ALERT_STATUS_RECOVERED && (flapping || flappingHistory.filter((f: boolean) => f).length > 0) ) - .map(({ event, flappingHistory, flapping: isCurrentlyFlapping, pendingRecoveredCount }) => { - const alertId = event[ALERT_INSTANCE_ID]!; - const alertUuid = event[ALERT_UUID]!; - const started = new Date(event[ALERT_START]!).toISOString(); - const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); - return [ - alertId, - { alertId, alertUuid, started, flappingHistory, flapping, pendingRecoveredCount }, - ]; - }) + .map( + ({ + event, + flappingHistory, + flapping: isCurrentlyFlapping, + pendingRecoveredCount, + activeCount, + }) => { + const alertId = event[ALERT_INSTANCE_ID]!; + const alertUuid = event[ALERT_UUID]!; + const started = new Date(event[ALERT_START]!).toISOString(); + const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); + return [ + alertId, + { + alertId, + alertUuid, + started, + flappingHistory, + flapping, + pendingRecoveredCount, + activeCount, + }, + ]; + } + ) ); return { diff --git a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts index b3047303bcb08f..9fa82fcf739ab5 100644 --- a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts @@ -20,6 +20,7 @@ describe('getAlertsForNotification', () => { }, flapping: true, pendingRecoveredCount: 3, + activeCount: 3, }; const alert2 = { event: { @@ -40,13 +41,32 @@ describe('getAlertsForNotification', () => { pendingRecoveredCount: 4, flappingHistory: [true, true], }; + const alert5 = { + event: { + 'kibana.alert.status': ALERT_STATUS_ACTIVE, + }, + activeCount: 1, + pendingRecoveredCount: 0, + flappingHistory: [], + }; test('should set pendingRecoveredCount to zero for all active alerts', () => { - const trackedEvents = [alert4]; - expect(getAlertsForNotification(DEFAULT_FLAPPING_SETTINGS, trackedEvents)) - .toMatchInlineSnapshot(` + const trackedEvents = cloneDeep([alert4]); + const newEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification('timestamp', DEFAULT_FLAPPING_SETTINGS, 0, trackedEvents, newEvents) + ).toMatchInlineSnapshot(` Array [ Object { + "activeCount": 2, + "event": Object { + "kibana.alert.status": "active", + }, + "flappingHistory": Array [], + "pendingRecoveredCount": 0, + }, + Object { + "activeCount": 1, "event": Object { "kibana.alert.status": "active", }, @@ -62,10 +82,11 @@ describe('getAlertsForNotification', () => { test('should not remove alerts if the num of recovered alerts is not at the limit', () => { const trackedEvents = cloneDeep([alert1, alert2, alert3]); - expect(getAlertsForNotification(DEFAULT_FLAPPING_SETTINGS, trackedEvents)) + expect(getAlertsForNotification('timestamp', DEFAULT_FLAPPING_SETTINGS, 0, trackedEvents, [])) .toMatchInlineSnapshot(` Array [ Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, @@ -73,12 +94,14 @@ describe('getAlertsForNotification', () => { "pendingRecoveredCount": 0, }, Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, "flapping": false, }, Object { + "activeCount": 0, "event": Object { "event.action": "active", "kibana.alert.status": "active", @@ -92,10 +115,11 @@ describe('getAlertsForNotification', () => { test('should reset counts and not modify alerts if flapping is disabled', () => { const trackedEvents = cloneDeep([alert1, alert2, alert3]); - expect(getAlertsForNotification(DISABLE_FLAPPING_SETTINGS, trackedEvents)) + expect(getAlertsForNotification('timestamp', DISABLE_FLAPPING_SETTINGS, 0, trackedEvents, [])) .toMatchInlineSnapshot(` Array [ Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, @@ -103,6 +127,7 @@ describe('getAlertsForNotification', () => { "pendingRecoveredCount": 0, }, Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, @@ -110,12 +135,107 @@ describe('getAlertsForNotification', () => { "pendingRecoveredCount": 0, }, Object { + "activeCount": 0, + "event": Object { + "kibana.alert.status": "recovered", + }, + "flapping": true, + "pendingRecoveredCount": 0, + }, + ] + `); + }); + + test('should increment activeCount for all active alerts', () => { + const trackedEvents = cloneDeep([alert4]); + const newEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification('timestamp', DEFAULT_FLAPPING_SETTINGS, 0, trackedEvents, newEvents) + ).toMatchInlineSnapshot(` + Array [ + Object { + "activeCount": 2, + "event": Object { + "kibana.alert.status": "active", + }, + "flappingHistory": Array [], + "pendingRecoveredCount": 0, + }, + Object { + "activeCount": 1, + "event": Object { + "kibana.alert.status": "active", + }, + "flappingHistory": Array [ + true, + true, + ], + "pendingRecoveredCount": 0, + }, + ] + `); + }); + + test('should reset activeCount for all recovered alerts', () => { + const trackedEvents = cloneDeep([alert1, alert2]); + expect(getAlertsForNotification('timestamp', DEFAULT_FLAPPING_SETTINGS, 0, trackedEvents, [])) + .toMatchInlineSnapshot(` + Array [ + Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, "flapping": true, "pendingRecoveredCount": 0, }, + Object { + "activeCount": 0, + "event": Object { + "kibana.alert.status": "recovered", + }, + "flapping": false, + }, + ] + `); + }); + + test('should not return active alerts if the activeCount is less than the rule alertDelay', () => { + const trackedEvents = cloneDeep([alert4]); + const newEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification('timestamp', DEFAULT_FLAPPING_SETTINGS, 5, trackedEvents, newEvents) + ).toMatchInlineSnapshot(`Array []`); + }); + + test('should update active alert to look like a new alert if the activeCount is equal to the rule alertDelay', () => { + const trackedEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification( + 'timestamp', + DEFAULT_FLAPPING_SETTINGS, + 2, + trackedEvents, + [], + ['maintenance-window-id'] + ) + ).toMatchInlineSnapshot(` + Array [ + Object { + "activeCount": 2, + "event": Object { + "event.action": "open", + "kibana.alert.maintenance_window_ids": Array [ + "maintenance-window-id", + ], + "kibana.alert.status": "active", + "kibana.alert.time_range": Object { + "gte": "timestamp", + }, + }, + "flappingHistory": Array [], + "pendingRecoveredCount": 0, + }, ] `); }); diff --git a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts index 878db2a9180227..c9ac8a954c2c65 100644 --- a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts +++ b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts @@ -12,31 +12,54 @@ import { ALERT_STATUS_ACTIVE, ALERT_STATUS_RECOVERED, EVENT_ACTION, + ALERT_TIME_RANGE, + ALERT_MAINTENANCE_WINDOW_IDS, } from '@kbn/rule-data-utils'; export function getAlertsForNotification( + timestamp: string, flappingSettings: RulesSettingsFlappingProperties, - trackedEventsToIndex: any[] + alertDelay: number, + trackedEventsToIndex: any[], + newEventsToIndex: any[], + maintenanceWindowIds?: string[] ) { - return trackedEventsToIndex.map((trackedEvent) => { - if (!flappingSettings.enabled || trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_ACTIVE) { + const events: any[] = []; + for (const trackedEvent of [...newEventsToIndex, ...trackedEventsToIndex]) { + if (trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_ACTIVE) { + const count = trackedEvent.activeCount || 0; + trackedEvent.activeCount = count + 1; trackedEvent.pendingRecoveredCount = 0; - } else if ( - flappingSettings.enabled && - trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_RECOVERED - ) { - if (trackedEvent.flapping) { - const count = trackedEvent.pendingRecoveredCount || 0; - trackedEvent.pendingRecoveredCount = count + 1; - if (trackedEvent.pendingRecoveredCount < flappingSettings.statusChangeThreshold) { - trackedEvent.event[ALERT_STATUS] = ALERT_STATUS_ACTIVE; - trackedEvent.event[EVENT_ACTION] = 'active'; - delete trackedEvent.event[ALERT_END]; - } else { - trackedEvent.pendingRecoveredCount = 0; + if (trackedEvent.activeCount < alertDelay) { + continue; + } else { + if (trackedEvent.activeCount === alertDelay) { + trackedEvent.event[ALERT_TIME_RANGE] = { gte: timestamp }; + trackedEvent.event[EVENT_ACTION] = 'open'; + if (maintenanceWindowIds?.length) { + trackedEvent.event[ALERT_MAINTENANCE_WINDOW_IDS] = maintenanceWindowIds; + } } } + } else if (trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_RECOVERED) { + trackedEvent.activeCount = 0; + if (flappingSettings.enabled) { + if (trackedEvent.flapping) { + const count = trackedEvent.pendingRecoveredCount || 0; + trackedEvent.pendingRecoveredCount = count + 1; + if (trackedEvent.pendingRecoveredCount < flappingSettings.statusChangeThreshold) { + trackedEvent.event[ALERT_STATUS] = ALERT_STATUS_ACTIVE; + trackedEvent.event[EVENT_ACTION] = 'active'; + delete trackedEvent.event[ALERT_END]; + } else { + trackedEvent.pendingRecoveredCount = 0; + } + } + } else { + trackedEvent.pendingRecoveredCount = 0; + } } - return trackedEvent; - }); + events.push(trackedEvent); + } + return events; } diff --git a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_form.tsx b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_form.tsx index e65f8c90bd03d6..6570e79a6f6c59 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_form.tsx +++ b/x-pack/plugins/triggers_actions_ui/public/application/sections/rule_form/rule_form.tsx @@ -834,7 +834,7 @@ export const RuleForm = ({ label={[ , } />,