From 1204165ef09c5aa4a4c9e70650dfd14f3061d13f Mon Sep 17 00:00:00 2001 From: Alexi Doak <109488926+doakalexi@users.noreply.github.com> Date: Tue, 6 Feb 2024 13:43:57 -0800 Subject: [PATCH] [ResponseOps] Alert creation delay based on user definition (#175851) Resolves https://github.com/elastic/kibana/issues/173009 ## Summary This PR: - Changes the field name from `notification_delay` to `alert_delay` - Updates the alerts client and rule registry to index new alert docs on a delay - Updates the framework code to delay the creation of an alert ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios ### To verify - Use [Dev Tools](http://localhost:5601/app/dev_tools#/console) to create a rule with the `alertDelay` ``` POST kbn:/api/alerting/rule { "params": { "searchType": "esQuery", "timeWindowSize": 5, "timeWindowUnit": "m", "threshold": [ -1 ], "thresholdComparator": ">", "size": 100, "esQuery": """{ "query":{ "match_all" : {} } }""", "aggType": "count", "groupBy": "all", "termSize": 5, "excludeHitsFromPreviousRun": false, "sourceFields": [], "index": [ ".kibana-event-log*" ], "timeField": "@timestamp" }, "consumer": "stackAlerts", "schedule": { "interval": "1m" }, "tags": [], "name": "test", "rule_type_id": ".es-query", "actions": [ { "group": "query matched", "id": "${ACTION_ID}", "params": { "level": "info", "message": """Elasticsearch query rule '{{rule.name}}' is active: - Value: {{context.value}} - Conditions Met: {{context.conditions}} over {{rule.params.timeWindowSize}}{{rule.params.timeWindowUnit}} - Timestamp: {{context.date}} - Link: {{context.link}}""" }, "frequency": { "notify_when": "onActionGroupChange", "throttle": null, "summary": false } } ], "alert_delay": { "active": 3 } } ``` - Verify that the alert will not be created until it has matched the delay threshold. - Verify that the delay does not affect recovered alerts --- .../src/lifecycle_state.ts | 2 + .../src/task_state/v1/schema.ts | 3 +- .../routes/rule/apis/create/schemas/v1.ts | 4 +- .../routes/rule/apis/create/types/v1.ts | 2 +- .../common/routes/rule/response/index.ts | 2 +- .../common/routes/rule/response/schemas/v1.ts | 4 +- .../common/routes/rule/response/types/v1.ts | 2 +- x-pack/plugins/alerting/common/rule.ts | 5 +- .../alerts_client/alerts_client.test.ts | 20 + .../server/alerts_client/alerts_client.ts | 5 + .../alerts_client/alerts_client_fixtures.ts | 1 + .../legacy_alerts_client.test.ts | 5 +- .../alerts_client/legacy_alerts_client.ts | 7 +- .../alerts_client/lib/format_rule.test.ts | 1 + .../alerting/server/alerts_client/types.ts | 3 + .../alerts_service/alerts_service.test.ts | 16 + .../create/schemas/create_rule_data_schema.ts | 8 +- .../methods/create/types/create_rule_data.ts | 2 +- .../server/application/rule/schemas/index.ts | 2 +- .../application/rule/schemas/rule_schemas.ts | 6 +- ...ransform_rule_attributes_to_rule_domain.ts | 2 +- .../transform_rule_domain_to_rule.ts | 2 +- ...ransform_rule_domain_to_rule_attributes.ts | 2 +- .../server/application/rule/types/rule.ts | 4 +- .../server/data/rule/types/rule_attributes.ts | 4 +- .../lib/get_alerts_for_notification.test.ts | 106 ++- .../server/lib/get_alerts_for_notification.ts | 21 +- .../transforms/transform_create_body/v1.ts | 2 +- .../transform_rule_to_rule_response/v1.ts | 2 +- .../saved_objects/schemas/raw_rule/v1.ts | 4 +- .../task_runner/execution_handler.test.ts | 159 ----- .../server/task_runner/execution_handler.ts | 16 - .../alerting/server/task_runner/fixtures.ts | 2 +- .../server/task_runner/task_runner.test.ts | 6 +- .../server/task_runner/task_runner.ts | 4 + .../task_runner_alerts_client.test.ts | 2 + x-pack/plugins/alerting/server/types.ts | 5 +- .../utils/create_lifecycle_executor.test.ts | 34 + .../server/utils/create_lifecycle_executor.ts | 89 ++- .../utils/get_alerts_for_notification.test.ts | 154 +++- .../utils/get_alerts_for_notification.ts | 72 +- .../get_updated_flapping_history.test.ts | 5 + .../tests/alerting/group1/event_log.ts | 139 ++-- .../{notification_delay.ts => alert_delay.ts} | 35 +- .../alerts_as_data_alert_delay.ts | 665 ++++++++++++++++++ .../alerting/group4/alerts_as_data/index.ts | 1 + .../tests/alerting/group4/index.ts | 2 +- 47 files changed, 1253 insertions(+), 386 deletions(-) rename x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/{notification_delay.ts => alert_delay.ts} (82%) create mode 100644 x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/alerts_as_data_alert_delay.ts diff --git a/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts index 8fa5cb111f6d91..f82791dd4e4ea1 100644 --- a/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts +++ b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts @@ -20,6 +20,8 @@ const trackedAlertStateRt = t.type({ // count of consecutive recovered alerts for flapping // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings pendingRecoveredCount: t.number, + // count of consecutive active alerts will reset if the alert is recovered + activeCount: t.number, }); export type TrackedLifecycleAlertState = t.TypeOf; diff --git a/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts b/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts index 247c0d7fc8d87f..247cc7124c000b 100644 --- a/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts +++ b/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts @@ -36,8 +36,7 @@ export const metaSchema = schema.object({ // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings pendingRecoveredCount: schema.maybe(schema.number()), uuid: schema.maybe(schema.string()), - // count of consecutive active alerts - // will reset if the alert is recovered or if equal to notificationDelay.active stored in the rule + // count of consecutive active alerts will reset if the alert is recovered activeCount: schema.maybe(schema.number()), }); diff --git a/x-pack/plugins/alerting/common/routes/rule/apis/create/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/rule/apis/create/schemas/v1.ts index bdc60a6562fe20..7d1a37b7c75f73 100644 --- a/x-pack/plugins/alerting/common/routes/rule/apis/create/schemas/v1.ts +++ b/x-pack/plugins/alerting/common/routes/rule/apis/create/schemas/v1.ts @@ -7,7 +7,7 @@ import { schema } from '@kbn/config-schema'; import { validateDurationV1, validateHoursV1, validateTimezoneV1 } from '../../../validation'; -import { notifyWhenSchemaV1, notificationDelaySchemaV1 } from '../../../response'; +import { notifyWhenSchemaV1, alertDelaySchemaV1 } from '../../../response'; import { alertsFilterQuerySchemaV1 } from '../../../../alerts_filter_query'; export const actionFrequencySchema = schema.object({ @@ -68,7 +68,7 @@ export const createBodySchema = schema.object({ }), actions: schema.arrayOf(actionSchema, { defaultValue: [] }), notify_when: schema.maybe(schema.nullable(notifyWhenSchemaV1)), - notification_delay: schema.maybe(notificationDelaySchemaV1), + alert_delay: schema.maybe(alertDelaySchemaV1), }); export const createParamsSchema = schema.object({ diff --git a/x-pack/plugins/alerting/common/routes/rule/apis/create/types/v1.ts b/x-pack/plugins/alerting/common/routes/rule/apis/create/types/v1.ts index 328f44fe185bf4..9d5d1e39525ab7 100644 --- a/x-pack/plugins/alerting/common/routes/rule/apis/create/types/v1.ts +++ b/x-pack/plugins/alerting/common/routes/rule/apis/create/types/v1.ts @@ -30,7 +30,7 @@ export interface CreateRuleRequestBody { schedule: CreateBodySchema['schedule']; actions: CreateBodySchema['actions']; notify_when?: CreateBodySchema['notify_when']; - notification_delay?: CreateBodySchema['notification_delay']; + alert_delay?: CreateBodySchema['alert_delay']; } export interface CreateRuleResponse { diff --git a/x-pack/plugins/alerting/common/routes/rule/response/index.ts b/x-pack/plugins/alerting/common/routes/rule/response/index.ts index 8e405c599e4835..8c784e744d4730 100644 --- a/x-pack/plugins/alerting/common/routes/rule/response/index.ts +++ b/x-pack/plugins/alerting/common/routes/rule/response/index.ts @@ -37,7 +37,7 @@ export { ruleSnoozeScheduleSchema as ruleSnoozeScheduleSchemaV1, notifyWhenSchema as notifyWhenSchemaV1, scheduleIdsSchema as scheduleIdsSchemaV1, - notificationDelaySchema as notificationDelaySchemaV1, + alertDelaySchema as alertDelaySchemaV1, } from './schemas/v1'; export type { diff --git a/x-pack/plugins/alerting/common/routes/rule/response/schemas/v1.ts b/x-pack/plugins/alerting/common/routes/rule/response/schemas/v1.ts index 67f57926d54603..f485aa7374d6cd 100644 --- a/x-pack/plugins/alerting/common/routes/rule/response/schemas/v1.ts +++ b/x-pack/plugins/alerting/common/routes/rule/response/schemas/v1.ts @@ -182,7 +182,7 @@ export const ruleSnoozeScheduleSchema = schema.object({ skipRecurrences: schema.maybe(schema.arrayOf(schema.string())), }); -export const notificationDelaySchema = schema.object({ +export const alertDelaySchema = schema.object({ active: schema.number(), }); @@ -218,7 +218,7 @@ export const ruleResponseSchema = schema.object({ revision: schema.number(), running: schema.maybe(schema.nullable(schema.boolean())), view_in_app_relative_url: schema.maybe(schema.nullable(schema.string())), - notification_delay: schema.maybe(notificationDelaySchema), + alert_delay: schema.maybe(alertDelaySchema), }); export const scheduleIdsSchema = schema.maybe(schema.arrayOf(schema.string())); diff --git a/x-pack/plugins/alerting/common/routes/rule/response/types/v1.ts b/x-pack/plugins/alerting/common/routes/rule/response/types/v1.ts index 453d1a96d24dcf..25e08ab407ebb6 100644 --- a/x-pack/plugins/alerting/common/routes/rule/response/types/v1.ts +++ b/x-pack/plugins/alerting/common/routes/rule/response/types/v1.ts @@ -53,5 +53,5 @@ export interface RuleResponse { revision: RuleResponseSchemaType['revision']; running?: RuleResponseSchemaType['running']; view_in_app_relative_url?: RuleResponseSchemaType['view_in_app_relative_url']; - notification_delay?: RuleResponseSchemaType['notification_delay']; + alert_delay?: RuleResponseSchemaType['alert_delay']; } diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts index 590c1d4312d57b..7cec5bdbdd7a62 100644 --- a/x-pack/plugins/alerting/common/rule.ts +++ b/x-pack/plugins/alerting/common/rule.ts @@ -141,7 +141,7 @@ export interface MappedParamsProperties { export type MappedParams = SavedObjectAttributes & MappedParamsProperties; -export interface NotificationDelay { +export interface AlertDelay extends SavedObjectAttributes { active: number; } @@ -178,7 +178,7 @@ export interface Rule { revision: number; running?: boolean | null; viewInAppRelativeUrl?: string; - notificationDelay?: NotificationDelay; + alertDelay?: AlertDelay; } export interface SanitizedAlertsFilter extends AlertsFilter { @@ -222,6 +222,7 @@ export type SanitizedRuleConfig = Pick< | 'muteAll' | 'revision' | 'snoozeSchedule' + | 'alertDelay' > & { producer: string; ruleTypeId: string; diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts index 0ef8f7c31e0193..81848c3e656865 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts @@ -308,6 +308,7 @@ describe('Alerts Client', () => { flappingSettings: DEFAULT_FLAPPING_SETTINGS, notifyOnActionGroupChange: true, maintenanceWindowIds: [], + alertDelay: 0, }; }); @@ -516,6 +517,25 @@ describe('Alerts Client', () => { }); }); + test('should not index new alerts if the activeCount is less than the rule alertDelay', async () => { + const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>({ + ...alertsClientParams, + rule: { ...alertsClientParams.rule, alertDelay: 3 }, + }); + + await alertsClient.initializeExecution(defaultExecutionOpts); + + // Report 1 new alerts + const alertExecutorService = alertsClient.factory(); + alertExecutorService.create('1').scheduleActions('default'); + + alertsClient.processAndLogAlerts(processAndLogAlertsOpts); + + await alertsClient.persistAlerts(); + + expect(clusterClient.bulk).not.toHaveBeenCalled(); + }); + test('should update ongoing alerts in existing index', async () => { clusterClient.search.mockResolvedValue({ took: 10, diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts index cc443975cee466..0c1eb897a6e595 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts @@ -344,6 +344,11 @@ export class AlertsClient< }) ); } else { + // skip writing the alert document if the number of consecutive + // active alerts is less than the rule alertDelay threshold + if (activeAlerts[id].getActiveCount() < this.options.rule.alertDelay) { + continue; + } activeAlertsToIndex.push( buildNewAlert< AlertData, diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts index 90c2f4c40b65bd..0da20a5e49b70f 100644 --- a/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts +++ b/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts @@ -25,6 +25,7 @@ export const alertRuleData: AlertRuleData = { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }; export const mockAAD = { diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts index 446ecfd79c8b80..d01060820ae524 100644 --- a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts @@ -245,6 +245,7 @@ describe('Legacy Alerts Client', () => { flappingSettings: DEFAULT_FLAPPING_SETTINGS, notifyOnActionGroupChange: true, maintenanceWindowIds: ['window-id1', 'window-id2'], + alertDelay: 5, }); expect(processAlerts).toHaveBeenCalledWith({ @@ -275,13 +276,15 @@ describe('Legacy Alerts Client', () => { }, true, 'default', + 5, {}, { '1': new Alert('1', testAlert1), '2': new Alert('2', testAlert2), }, {}, - {} + {}, + null ); expect(logAlerts).toHaveBeenCalledWith({ diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts index 0de97a8a29be68..a5cfc642a19ee8 100644 --- a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts +++ b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts @@ -140,6 +140,7 @@ export class LegacyAlertsClient< notifyOnActionGroupChange, flappingSettings, maintenanceWindowIds, + alertDelay, }: ProcessAlertsOpts) { const { newAlerts: processedAlertsNew, @@ -168,10 +169,12 @@ export class LegacyAlertsClient< flappingSettings, notifyOnActionGroupChange, this.options.ruleType.defaultActionGroupId, + alertDelay, processedAlertsNew, processedAlertsActive, trimmedAlertsRecovered, - processedAlertsRecoveredCurrent + processedAlertsRecoveredCurrent, + this.startedAtString ); alerts.currentRecoveredAlerts = merge(alerts.currentRecoveredAlerts, earlyRecoveredAlerts); @@ -203,11 +206,13 @@ export class LegacyAlertsClient< flappingSettings, notifyOnActionGroupChange, maintenanceWindowIds, + alertDelay, }: ProcessAndLogAlertsOpts) { this.processAlerts({ notifyOnActionGroupChange, flappingSettings, maintenanceWindowIds, + alertDelay, }); this.logAlerts({ diff --git a/x-pack/plugins/alerting/server/alerts_client/lib/format_rule.test.ts b/x-pack/plugins/alerting/server/alerts_client/lib/format_rule.test.ts index 9f335f8266b214..c9cc7fbefbfa41 100644 --- a/x-pack/plugins/alerting/server/alerts_client/lib/format_rule.test.ts +++ b/x-pack/plugins/alerting/server/alerts_client/lib/format_rule.test.ts @@ -61,6 +61,7 @@ describe('formatRule', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, ruleType, }) diff --git a/x-pack/plugins/alerting/server/alerts_client/types.ts b/x-pack/plugins/alerting/server/alerts_client/types.ts index 44690d58811794..1de1c06f5e826b 100644 --- a/x-pack/plugins/alerting/server/alerts_client/types.ts +++ b/x-pack/plugins/alerting/server/alerts_client/types.ts @@ -47,6 +47,7 @@ export interface AlertRuleData { revision: number; spaceId: string; tags: string[]; + alertDelay: number; } export interface AlertRule { @@ -111,12 +112,14 @@ export interface ProcessAndLogAlertsOpts { flappingSettings: RulesSettingsFlappingProperties; notifyOnActionGroupChange: boolean; maintenanceWindowIds: string[]; + alertDelay: number; } export interface ProcessAlertsOpts { flappingSettings: RulesSettingsFlappingProperties; notifyOnActionGroupChange: boolean; maintenanceWindowIds: string[]; + alertDelay: number; } export interface LogAlertsOpts { diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts index b73f671ab0695c..def19570f7e0a5 100644 --- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts @@ -1475,6 +1475,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); @@ -1495,6 +1496,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, kibanaVersion: '8.8.0', }); @@ -1528,6 +1530,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); @@ -1576,6 +1579,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); @@ -1610,6 +1614,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, kibanaVersion: '8.8.0', }); @@ -1674,6 +1679,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }), alertsService.createAlertsClient({ @@ -1691,6 +1697,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }), ]); @@ -1725,6 +1732,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, kibanaVersion: '8.8.0', }); @@ -1781,6 +1789,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); @@ -1801,6 +1810,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, kibanaVersion: '8.8.0', }); @@ -1865,6 +1875,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); }; @@ -1892,6 +1903,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, kibanaVersion: '8.8.0', }); @@ -1961,6 +1973,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); }; @@ -2026,6 +2039,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); @@ -2091,6 +2105,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); @@ -2154,6 +2169,7 @@ describe('Alerts Service', () => { revision: 0, spaceId: 'default', tags: ['rule-', '-tags'], + alertDelay: 0, }, }); diff --git a/x-pack/plugins/alerting/server/application/rule/methods/create/schemas/create_rule_data_schema.ts b/x-pack/plugins/alerting/server/application/rule/methods/create/schemas/create_rule_data_schema.ts index 44a89e05992fe3..9d1823381d17d6 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/create/schemas/create_rule_data_schema.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/create/schemas/create_rule_data_schema.ts @@ -7,11 +7,7 @@ import { schema } from '@kbn/config-schema'; import { validateDuration } from '../../../validation'; -import { - notifyWhenSchema, - actionAlertsFilterSchema, - notificationDelaySchema, -} from '../../../schemas'; +import { notifyWhenSchema, actionAlertsFilterSchema, alertDelaySchema } from '../../../schemas'; export const createRuleDataSchema = schema.object({ name: schema.string(), @@ -44,5 +40,5 @@ export const createRuleDataSchema = schema.object({ { defaultValue: [] } ), notifyWhen: schema.maybe(schema.nullable(notifyWhenSchema)), - notificationDelay: schema.maybe(notificationDelaySchema), + alertDelay: schema.maybe(alertDelaySchema), }); diff --git a/x-pack/plugins/alerting/server/application/rule/methods/create/types/create_rule_data.ts b/x-pack/plugins/alerting/server/application/rule/methods/create/types/create_rule_data.ts index f99beda90e80ae..abee30ec9a5248 100644 --- a/x-pack/plugins/alerting/server/application/rule/methods/create/types/create_rule_data.ts +++ b/x-pack/plugins/alerting/server/application/rule/methods/create/types/create_rule_data.ts @@ -22,5 +22,5 @@ export interface CreateRuleData { schedule: CreateRuleDataType['schedule']; actions: CreateRuleDataType['actions']; notifyWhen?: CreateRuleDataType['notifyWhen']; - notificationDelay?: CreateRuleDataType['notificationDelay']; + alertDelay?: CreateRuleDataType['alertDelay']; } diff --git a/x-pack/plugins/alerting/server/application/rule/schemas/index.ts b/x-pack/plugins/alerting/server/application/rule/schemas/index.ts index d039d190f1e96b..06645e90d7baff 100644 --- a/x-pack/plugins/alerting/server/application/rule/schemas/index.ts +++ b/x-pack/plugins/alerting/server/application/rule/schemas/index.ts @@ -13,7 +13,7 @@ export { monitoringSchema, ruleSchema, ruleDomainSchema, - notificationDelaySchema, + alertDelaySchema, } from './rule_schemas'; export { diff --git a/x-pack/plugins/alerting/server/application/rule/schemas/rule_schemas.ts b/x-pack/plugins/alerting/server/application/rule/schemas/rule_schemas.ts index b75a6e4f76aade..6041e475daf527 100644 --- a/x-pack/plugins/alerting/server/application/rule/schemas/rule_schemas.ts +++ b/x-pack/plugins/alerting/server/application/rule/schemas/rule_schemas.ts @@ -132,7 +132,7 @@ export const snoozeScheduleSchema = schema.object({ skipRecurrences: schema.maybe(schema.arrayOf(schema.string())), }); -export const notificationDelaySchema = schema.object({ +export const alertDelaySchema = schema.object({ active: schema.number(), }); @@ -172,7 +172,7 @@ export const ruleDomainSchema = schema.object({ revision: schema.number(), running: schema.maybe(schema.nullable(schema.boolean())), viewInAppRelativeUrl: schema.maybe(schema.nullable(schema.string())), - notificationDelay: schema.maybe(notificationDelaySchema), + alertDelay: schema.maybe(alertDelaySchema), }); /** @@ -210,5 +210,5 @@ export const ruleSchema = schema.object({ revision: schema.number(), running: schema.maybe(schema.nullable(schema.boolean())), viewInAppRelativeUrl: schema.maybe(schema.nullable(schema.string())), - notificationDelay: schema.maybe(notificationDelaySchema), + alertDelay: schema.maybe(alertDelaySchema), }); diff --git a/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts b/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts index ea331cec4fcc31..9cc8131b1fa2d6 100644 --- a/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts +++ b/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts @@ -216,7 +216,7 @@ export const transformRuleAttributesToRuleDomain = ( revision: ruleDomain.revision, running: ruleDomain.running, viewInAppRelativeUrl: ruleDomain.viewInAppRelativeUrl, - notificationDelay: ruleDomain.notificationDelay, + alertDelay: ruleDomain.alertDelay, }; if (isPublic) { diff --git a/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_domain_to_rule_attributes.ts b/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_domain_to_rule_attributes.ts index 2eb21c42587d3f..e527bd6f31df0e 100644 --- a/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_domain_to_rule_attributes.ts +++ b/x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_domain_to_rule_attributes.ts @@ -68,6 +68,6 @@ export const transformRuleDomainToRuleAttributes = ( ...(rule.nextRun !== undefined ? { nextRun: rule.nextRun?.toISOString() || null } : {}), revision: rule.revision, ...(rule.running !== undefined ? { running: rule.running } : {}), - ...(rule.notificationDelay !== undefined ? { notificationDelay: rule.notificationDelay } : {}), + ...(rule.alertDelay !== undefined ? { alertDelay: rule.alertDelay } : {}), }; }; diff --git a/x-pack/plugins/alerting/server/application/rule/types/rule.ts b/x-pack/plugins/alerting/server/application/rule/types/rule.ts index 1a7e7e1d371182..f59056b382440f 100644 --- a/x-pack/plugins/alerting/server/application/rule/types/rule.ts +++ b/x-pack/plugins/alerting/server/application/rule/types/rule.ts @@ -85,7 +85,7 @@ export interface Rule { revision: RuleSchemaType['revision']; running?: RuleSchemaType['running']; viewInAppRelativeUrl?: RuleSchemaType['viewInAppRelativeUrl']; - notificationDelay?: RuleSchemaType['notificationDelay']; + alertDelay?: RuleSchemaType['alertDelay']; } export interface RuleDomain { @@ -121,5 +121,5 @@ export interface RuleDomain { revision: RuleDomainSchemaType['revision']; running?: RuleDomainSchemaType['running']; viewInAppRelativeUrl?: RuleDomainSchemaType['viewInAppRelativeUrl']; - notificationDelay?: RuleSchemaType['notificationDelay']; + alertDelay?: RuleSchemaType['alertDelay']; } diff --git a/x-pack/plugins/alerting/server/data/rule/types/rule_attributes.ts b/x-pack/plugins/alerting/server/data/rule/types/rule_attributes.ts index 316578149c5ca8..e047be1b9cddf5 100644 --- a/x-pack/plugins/alerting/server/data/rule/types/rule_attributes.ts +++ b/x-pack/plugins/alerting/server/data/rule/types/rule_attributes.ts @@ -142,7 +142,7 @@ interface RuleMetaAttributes { versionApiKeyLastmodified?: string; } -interface NotificationDelayAttributes { +interface AlertDelayAttributes { active: number; } @@ -178,5 +178,5 @@ export interface RuleAttributes { nextRun?: string | null; revision: number; running?: boolean | null; - notificationDelay?: NotificationDelayAttributes; + alertDelay?: AlertDelayAttributes; } diff --git a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts index c1465d5b7a2388..4656f4377f1301 100644 --- a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts +++ b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts @@ -21,6 +21,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, { '1': alert1, }, @@ -89,6 +90,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, {}, {}, { @@ -222,6 +224,7 @@ describe('getAlertsForNotification', () => { DISABLE_FLAPPING_SETTINGS, true, 'default', + 0, {}, {}, { @@ -353,6 +356,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, false, 'default', + 0, {}, {}, { @@ -455,10 +459,11 @@ describe('getAlertsForNotification', () => { }); const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } }); - const { newAlerts, activeAlerts } = getAlertsForNotification( + const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification( DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, { '1': alert1, }, @@ -507,6 +512,30 @@ describe('getAlertsForNotification', () => { }, } `); + expect(currentActiveAlerts).toMatchInlineSnapshot(` + Object { + "1": Object { + "meta": Object { + "activeCount": 2, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-1", + }, + "state": Object {}, + }, + "2": Object { + "meta": Object { + "activeCount": 1, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-2", + }, + "state": Object {}, + }, + } + `); }); test('should reset activeCount for all recovered alerts', () => { @@ -517,6 +546,7 @@ describe('getAlertsForNotification', () => { DEFAULT_FLAPPING_SETTINGS, true, 'default', + 0, {}, {}, { @@ -574,4 +604,78 @@ describe('getAlertsForNotification', () => { } `); }); + + test('should remove the alert from newAlerts and should not return the alert in currentActiveAlerts if the activeCount is less than the rule alertDelay', () => { + const alert1 = new Alert('1', { + meta: { activeCount: 1, uuid: 'uuid-1' }, + }); + const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } }); + + const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification( + DEFAULT_FLAPPING_SETTINGS, + true, + 'default', + 5, + { + '1': alert1, + }, + { + '1': alert1, + '2': alert2, + }, + {}, + {} + ); + expect(newAlerts).toMatchInlineSnapshot(`Object {}`); + expect(activeAlerts).toMatchInlineSnapshot(` + Object { + "1": Object { + "meta": Object { + "activeCount": 2, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-1", + }, + "state": Object {}, + }, + "2": Object { + "meta": Object { + "activeCount": 1, + "flappingHistory": Array [], + "maintenanceWindowIds": Array [], + "pendingRecoveredCount": 0, + "uuid": "uuid-2", + }, + "state": Object {}, + }, + } + `); + expect(currentActiveAlerts).toMatchInlineSnapshot(`Object {}`); + }); + + test('should update active alert to look like a new alert if the activeCount is equal to the rule alertDelay', () => { + const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } }); + + const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification( + DEFAULT_FLAPPING_SETTINGS, + true, + 'default', + 1, + {}, + { + '2': alert2, + }, + {}, + {} + ); + expect(newAlerts['2'].getState().duration).toBe('0'); + expect(newAlerts['2'].getState().start).toBeTruthy(); + + expect(activeAlerts['2'].getState().duration).toBe('0'); + expect(activeAlerts['2'].getState().start).toBeTruthy(); + + expect(currentActiveAlerts['2'].getState().duration).toBe('0'); + expect(currentActiveAlerts['2'].getState().start).toBeTruthy(); + }); }); diff --git a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts index 63e95402549a4a..593d92b35383ba 100644 --- a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts +++ b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts @@ -19,10 +19,12 @@ export function getAlertsForNotification< flappingSettings: RulesSettingsFlappingProperties, notifyOnActionGroupChange: boolean, actionGroupId: string, + alertDelay: number, newAlerts: Record> = {}, activeAlerts: Record> = {}, recoveredAlerts: Record> = {}, - currentRecoveredAlerts: Record> = {} + currentRecoveredAlerts: Record> = {}, + startedAt?: string | null ) { const currentActiveAlerts: Record> = {}; @@ -30,7 +32,22 @@ export function getAlertsForNotification< const alert = activeAlerts[id]; alert.incrementActiveCount(); alert.resetPendingRecoveredCount(); - currentActiveAlerts[id] = alert; + // do not trigger an action notification if the number of consecutive + // active alerts is less than the rule alertDelay threshold + if (alert.getActiveCount() < alertDelay) { + // remove from new alerts + delete newAlerts[id]; + } else { + currentActiveAlerts[id] = alert; + // if the active count is equal to the alertDelay it is considered a new alert + if (alert.getActiveCount() === alertDelay) { + const currentTime = startedAt ?? new Date().toISOString(); + const state = alert.getState(); + // keep the state and update the start time and duration + alert.replaceState({ ...state, start: currentTime, duration: '0' }); + newAlerts[id] = alert; + } + } } for (const id of keys(currentRecoveredAlerts)) { diff --git a/x-pack/plugins/alerting/server/routes/rule/apis/create/transforms/transform_create_body/v1.ts b/x-pack/plugins/alerting/server/routes/rule/apis/create/transforms/transform_create_body/v1.ts index fcf92b386aaa2b..5dea295c40ed7b 100644 --- a/x-pack/plugins/alerting/server/routes/rule/apis/create/transforms/transform_create_body/v1.ts +++ b/x-pack/plugins/alerting/server/routes/rule/apis/create/transforms/transform_create_body/v1.ts @@ -58,6 +58,6 @@ export const transformCreateBody = ( schedule: createBody.schedule, actions: transformCreateBodyActions(createBody.actions), ...(createBody.notify_when ? { notifyWhen: createBody.notify_when } : {}), - ...(createBody.notification_delay ? { notificationDelay: createBody.notification_delay } : {}), + ...(createBody.alert_delay ? { alertDelay: createBody.alert_delay } : {}), }; }; diff --git a/x-pack/plugins/alerting/server/routes/rule/transforms/transform_rule_to_rule_response/v1.ts b/x-pack/plugins/alerting/server/routes/rule/transforms/transform_rule_to_rule_response/v1.ts index 4f55401160ce9f..f4fdedfc6f436e 100644 --- a/x-pack/plugins/alerting/server/routes/rule/transforms/transform_rule_to_rule_response/v1.ts +++ b/x-pack/plugins/alerting/server/routes/rule/transforms/transform_rule_to_rule_response/v1.ts @@ -119,5 +119,5 @@ export const transformRuleToRuleResponse = ( ...(rule.viewInAppRelativeUrl !== undefined ? { view_in_app_relative_url: rule.viewInAppRelativeUrl } : {}), - ...(rule.notificationDelay !== undefined ? { notification_delay: rule.notificationDelay } : {}), + ...(rule.alertDelay !== undefined ? { alert_delay: rule.alertDelay } : {}), }); diff --git a/x-pack/plugins/alerting/server/saved_objects/schemas/raw_rule/v1.ts b/x-pack/plugins/alerting/server/saved_objects/schemas/raw_rule/v1.ts index 495c2493f2e434..e0641e9b275ea6 100644 --- a/x-pack/plugins/alerting/server/saved_objects/schemas/raw_rule/v1.ts +++ b/x-pack/plugins/alerting/server/saved_objects/schemas/raw_rule/v1.ts @@ -213,7 +213,7 @@ const rawRuleActionSchema = schema.object({ useAlertDataForTemplate: schema.maybe(schema.boolean()), }); -export const notificationDelaySchema = schema.object({ +export const alertDelaySchema = schema.object({ active: schema.number(), }); @@ -274,5 +274,5 @@ export const rawRuleSchema = schema.object({ ), params: schema.recordOf(schema.string(), schema.maybe(schema.any())), typeVersion: schema.maybe(schema.number()), - notificationDelay: schema.maybe(notificationDelaySchema), + alertDelay: schema.maybe(alertDelaySchema), }); diff --git a/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts b/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts index 8f5147ea4de309..a742afb152b198 100644 --- a/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/execution_handler.test.ts @@ -2052,165 +2052,6 @@ describe('Execution Handler', () => { `); }); - test('does not schedule actions for alerts with activeCount less than the notificationDelay.active threshold', async () => { - const executionHandler = new ExecutionHandler( - generateExecutionParams({ - ...defaultExecutionParams, - rule: { - ...defaultExecutionParams.rule, - notificationDelay: { - active: 3, - }, - }, - }) - ); - - await executionHandler.run({ - ...generateAlert({ id: 1 }), - ...generateAlert({ id: 2, activeCount: 2 }), - }); - - expect(actionsClient.bulkEnqueueExecution).not.toHaveBeenCalled(); - expect(defaultExecutionParams.logger.debug).toHaveBeenCalledTimes(2); - - expect(defaultExecutionParams.logger.debug).toHaveBeenCalledWith( - 'no scheduling of action "1" for rule "1": the alert activeCount: 0 is less than the rule notificationDelay.active: 3 threshold.' - ); - expect(defaultExecutionParams.logger.debug).toHaveBeenCalledWith( - 'no scheduling of action "1" for rule "1": the alert activeCount: 2 is less than the rule notificationDelay.active: 3 threshold.' - ); - }); - - test('schedules actions for alerts with activeCount greater than or equal the notificationDelay.active threshold', async () => { - const executionHandler = new ExecutionHandler( - generateExecutionParams({ - ...defaultExecutionParams, - rule: { - ...defaultExecutionParams.rule, - notificationDelay: { - active: 3, - }, - }, - }) - ); - - await executionHandler.run({ - ...generateAlert({ id: 1, activeCount: 3 }), - ...generateAlert({ id: 2, activeCount: 4 }), - }); - - expect(actionsClient.bulkEnqueueExecution).toHaveBeenCalledTimes(1); - expect(actionsClient.bulkEnqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "actionTypeId": "test", - "apiKey": "MTIzOmFiYw==", - "consumer": "rule-consumer", - "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28", - "id": "1", - "params": Object { - "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 1 goes here", - "contextVal": "My goes here", - "foo": true, - "stateVal": "My goes here", - }, - "relatedSavedObjects": Array [ - Object { - "id": "1", - "namespace": "test1", - "type": "alert", - "typeId": "test", - }, - ], - "source": Object { - "source": Object { - "id": "1", - "type": "alert", - }, - "type": "SAVED_OBJECT", - }, - "spaceId": "test1", - }, - Object { - "actionTypeId": "test", - "apiKey": "MTIzOmFiYw==", - "consumer": "rule-consumer", - "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28", - "id": "1", - "params": Object { - "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 2 goes here", - "contextVal": "My goes here", - "foo": true, - "stateVal": "My goes here", - }, - "relatedSavedObjects": Array [ - Object { - "id": "1", - "namespace": "test1", - "type": "alert", - "typeId": "test", - }, - ], - "source": Object { - "source": Object { - "id": "1", - "type": "alert", - }, - "type": "SAVED_OBJECT", - }, - "spaceId": "test1", - }, - ], - ] - `); - }); - - test('schedules actions if notificationDelay.active threshold is not defined', async () => { - const executionHandler = new ExecutionHandler(generateExecutionParams()); - - await executionHandler.run({ - ...generateAlert({ id: 1, activeCount: 1 }), - }); - - expect(actionsClient.bulkEnqueueExecution).toHaveBeenCalledTimes(1); - expect(actionsClient.bulkEnqueueExecution.mock.calls[0]).toMatchInlineSnapshot(` - Array [ - Array [ - Object { - "actionTypeId": "test", - "apiKey": "MTIzOmFiYw==", - "consumer": "rule-consumer", - "executionId": "5f6aa57d-3e22-484e-bae8-cbed868f4d28", - "id": "1", - "params": Object { - "alertVal": "My 1 name-of-alert test1 tag-A,tag-B 1 goes here", - "contextVal": "My goes here", - "foo": true, - "stateVal": "My goes here", - }, - "relatedSavedObjects": Array [ - Object { - "id": "1", - "namespace": "test1", - "type": "alert", - "typeId": "test", - }, - ], - "source": Object { - "source": Object { - "id": "1", - "type": "alert", - }, - "type": "SAVED_OBJECT", - }, - "spaceId": "test1", - }, - ], - ] - `); - }); - describe('rule url', () => { const ruleWithUrl = { ...rule, diff --git a/x-pack/plugins/alerting/server/task_runner/execution_handler.ts b/x-pack/plugins/alerting/server/task_runner/execution_handler.ts index e118b4d327ce18..ec690bb8ba0f5f 100644 --- a/x-pack/plugins/alerting/server/task_runner/execution_handler.ts +++ b/x-pack/plugins/alerting/server/task_runner/execution_handler.ts @@ -628,22 +628,6 @@ export class ExecutionHandler< continue; } - if ( - this.rule.notificationDelay && - alert.getActiveCount() < this.rule.notificationDelay.active - ) { - this.logger.debug( - `no scheduling of action "${action.id}" for rule "${ - this.taskInstance.params.alertId - }": the alert activeCount: ${alert.getActiveCount()} is less than the rule notificationDelay.active: ${ - this.rule.notificationDelay.active - } threshold.` - ); - continue; - } else { - alert.resetActiveCount(); - } - const actionGroup = this.getActionGroup(alert); if (!this.ruleTypeActionGroups!.has(actionGroup)) { diff --git a/x-pack/plugins/alerting/server/task_runner/fixtures.ts b/x-pack/plugins/alerting/server/task_runner/fixtures.ts index 3d647966414f5c..b2a984ea5768fa 100644 --- a/x-pack/plugins/alerting/server/task_runner/fixtures.ts +++ b/x-pack/plugins/alerting/server/task_runner/fixtures.ts @@ -454,7 +454,7 @@ export const generateAlertInstance = ( flapping: false, maintenanceWindowIds: maintenanceWindowIds || [], pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { bar: false, diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts index d14d44010252d8..e4afa351d4f146 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.test.ts @@ -2954,7 +2954,7 @@ describe('Task Runner', () => { maintenanceWindowIds: [], flapping: false, pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { duration: '0', @@ -3125,7 +3125,7 @@ describe('Task Runner', () => { maintenanceWindowIds: [], flapping: false, pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { duration: '0', @@ -3143,7 +3143,7 @@ describe('Task Runner', () => { maintenanceWindowIds: [], flapping: false, pendingRecoveredCount: 0, - activeCount: 0, + activeCount: 1, }, state: { duration: '0', diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner.ts b/x-pack/plugins/alerting/server/task_runner/task_runner.ts index 208e46b88a1f66..93f655965e92a2 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner.ts @@ -257,6 +257,7 @@ export class TaskRunner< revision: rule.revision, spaceId, tags: rule.tags, + alertDelay: rule.alertDelay?.active ?? 0, }; } @@ -311,6 +312,7 @@ export class TaskRunner< muteAll, revision, snoozeSchedule, + alertDelay, } = rule; const { params: { alertId: ruleId, spaceId }, @@ -525,6 +527,7 @@ export class TaskRunner< notifyWhen, muteAll, snoozeSchedule, + alertDelay, }, logger: this.logger, flappingSettings, @@ -582,6 +585,7 @@ export class TaskRunner< notifyWhen === RuleNotifyWhen.CHANGE || some(actions, (action) => action.frequency?.notifyWhen === RuleNotifyWhen.CHANGE), maintenanceWindowIds: maintenanceWindowsWithoutScopedQueryIds, + alertDelay: alertDelay?.active ?? 0, }); }); diff --git a/x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts b/x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts index ebf81f5bf050ec..bd47acbbdb8c13 100644 --- a/x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts +++ b/x-pack/plugins/alerting/server/task_runner/task_runner_alerts_client.test.ts @@ -302,6 +302,7 @@ describe('Task Runner', () => { ruleType: ruleTypeWithAlerts, namespace: 'default', rule: { + alertDelay: 0, consumer: 'bar', executionId: '5f6aa57d-3e22-484e-bae8-cbed868f4d28', id: '1', @@ -800,6 +801,7 @@ describe('Task Runner', () => { expect(alertsClientToUse.processAlerts).toHaveBeenCalledWith({ notifyOnActionGroupChange: false, + alertDelay: 0, flappingSettings: { enabled: true, lookBackWindow: 20, diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index ff16cfb593c282..eeb13576ce39d5 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -62,7 +62,7 @@ import { AlertsFilter, AlertsFilterTimeframe, RuleAlertData, - NotificationDelay, + AlertDelay, } from '../common'; import { PublicAlertFactory } from './alert/create_alert_factory'; import { RulesSettingsFlappingProperties } from '../common/rules_settings'; @@ -423,7 +423,6 @@ export type PublicRuleResultService = PublicLastRunSetters; export interface RawRuleLastRun extends SavedObjectAttributes, RuleLastRun {} export interface RawRuleMonitoring extends SavedObjectAttributes, RuleMonitoring {} -export interface RawNotificationDelay extends SavedObjectAttributes, NotificationDelay {} export interface RawRuleAlertsFilter extends AlertsFilter { query?: { @@ -500,7 +499,7 @@ export interface RawRule extends SavedObjectAttributes { nextRun?: string | null; revision: number; running?: boolean | null; - notificationDelay?: RawNotificationDelay; + alertDelay?: AlertDelay; } export type { DataStreamAdapter } from './alerts_service/lib/data_stream_adapter'; diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts index afed418ec8d3da..2367167b496971 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts @@ -208,6 +208,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -216,6 +217,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -358,6 +360,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -366,6 +369,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -490,6 +494,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -498,6 +503,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -732,6 +738,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -740,6 +747,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -850,6 +858,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -858,6 +867,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlerts: {}, @@ -963,6 +973,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -971,6 +982,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1077,6 +1089,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: { @@ -1087,6 +1100,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, }, @@ -1258,6 +1272,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1266,6 +1281,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1388,6 +1404,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1396,6 +1413,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1570,6 +1588,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: flapping, flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1578,6 +1597,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [false, false], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_2: { alertId: 'TEST_ALERT_2', @@ -1586,6 +1606,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: flapping, flapping: true, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_3: { alertId: 'TEST_ALERT_3', @@ -1594,6 +1615,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [false, false], flapping: true, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1604,6 +1626,7 @@ describe('createLifecycleExecutor', () => { expect(serializedAlerts.state.trackedAlerts).toEqual({ TEST_ALERT_0: { + activeCount: 1, alertId: 'TEST_ALERT_0', alertUuid: 'TEST_ALERT_0_UUID', flapping: true, @@ -1612,6 +1635,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-01T12:00:00.000Z', }, TEST_ALERT_1: { + activeCount: 1, alertId: 'TEST_ALERT_1', alertUuid: 'TEST_ALERT_1_UUID', flapping: false, @@ -1620,6 +1644,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-02T12:00:00.000Z', }, TEST_ALERT_2: { + activeCount: 1, alertId: 'TEST_ALERT_2', alertUuid: 'TEST_ALERT_2_UUID', flapping: true, @@ -1628,6 +1653,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-01T12:00:00.000Z', }, TEST_ALERT_3: { + activeCount: 1, alertId: 'TEST_ALERT_3', alertUuid: 'TEST_ALERT_3_UUID', flapping: true, @@ -1786,6 +1812,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [true, true, true, true], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_1: { alertId: 'TEST_ALERT_1', @@ -1794,6 +1821,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: notFlapping, flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_2: { alertId: 'TEST_ALERT_2', @@ -1802,6 +1830,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: [true, true], flapping: true, pendingRecoveredCount: 0, + activeCount: 0, }, TEST_ALERT_3: { alertId: 'TEST_ALERT_3', @@ -1810,6 +1839,7 @@ describe('createLifecycleExecutor', () => { flappingHistory: notFlapping, flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -1820,6 +1850,7 @@ describe('createLifecycleExecutor', () => { expect(serializedAlerts.state.trackedAlerts).toEqual({ TEST_ALERT_2: { + activeCount: 0, alertId: 'TEST_ALERT_2', alertUuid: 'TEST_ALERT_2_UUID', flapping: true, @@ -1831,6 +1862,7 @@ describe('createLifecycleExecutor', () => { expect(serializedAlerts.state.trackedAlertsRecovered).toEqual({ TEST_ALERT_0: { + activeCount: 0, alertId: 'TEST_ALERT_0', alertUuid: 'TEST_ALERT_0_UUID', flapping: true, @@ -1839,6 +1871,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-01T12:00:00.000Z', }, TEST_ALERT_1: { + activeCount: 0, alertId: 'TEST_ALERT_1', alertUuid: 'TEST_ALERT_1_UUID', flapping: false, @@ -1847,6 +1880,7 @@ describe('createLifecycleExecutor', () => { started: '2020-01-02T12:00:00.000Z', }, TEST_ALERT_3: { + activeCount: 0, alertId: 'TEST_ALERT_3', alertUuid: 'TEST_ALERT_3_UUID', flapping: false, diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 91d30fae7b3dc9..4bd3b912ae67df 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -290,7 +290,7 @@ export const createLifecycleExecutor = trackedAlertRecoveredIds ); - const { alertUuid, started, flapping, pendingRecoveredCount } = !isNew + const { alertUuid, started, flapping, pendingRecoveredCount, activeCount } = !isNew ? state.trackedAlerts[alertId] : { alertUuid: lifecycleAlertServices.getAlertUuid(alertId), @@ -299,6 +299,7 @@ export const createLifecycleExecutor = ? state.trackedAlertsRecovered[alertId].flapping : false, pendingRecoveredCount: 0, + activeCount: 0, }; const event: ParsedTechnicalFields & ParsedExperimentalFields = { @@ -342,16 +343,20 @@ export const createLifecycleExecutor = flappingHistory, flapping, pendingRecoveredCount, + activeCount, }; }); const trackedEventsToIndex = makeEventsDataMapFor(trackedAlertIds); const newEventsToIndex = makeEventsDataMapFor(newAlertIds); const trackedRecoveredEventsToIndex = makeEventsDataMapFor(trackedAlertRecoveredIds); - const allEventsToIndex = [ - ...getAlertsForNotification(flappingSettings, trackedEventsToIndex), - ...newEventsToIndex, - ]; + const allEventsToIndex = getAlertsForNotification( + flappingSettings, + rule.alertDelay?.active ?? 0, + trackedEventsToIndex, + newEventsToIndex, + { maintenanceWindowIds, timestamp: commonRuleFields[TIMESTAMP] } + ); // Only write alerts if: // - writing is enabled @@ -392,18 +397,34 @@ export const createLifecycleExecutor = } const nextTrackedAlerts = Object.fromEntries( - allEventsToIndex + [...newEventsToIndex, ...trackedEventsToIndex] .filter(({ event }) => event[ALERT_STATUS] !== ALERT_STATUS_RECOVERED) - .map(({ event, flappingHistory, flapping: isCurrentlyFlapping, pendingRecoveredCount }) => { - const alertId = event[ALERT_INSTANCE_ID]!; - const alertUuid = event[ALERT_UUID]!; - const started = new Date(event[ALERT_START]!).toISOString(); - const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); - return [ - alertId, - { alertId, alertUuid, started, flappingHistory, flapping, pendingRecoveredCount }, - ]; - }) + .map( + ({ + event, + flappingHistory, + flapping: isCurrentlyFlapping, + pendingRecoveredCount, + activeCount, + }) => { + const alertId = event[ALERT_INSTANCE_ID]!; + const alertUuid = event[ALERT_UUID]!; + const started = new Date(event[ALERT_START]!).toISOString(); + const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); + return [ + alertId, + { + alertId, + alertUuid, + started, + flappingHistory, + flapping, + pendingRecoveredCount, + activeCount, + }, + ]; + } + ) ); const nextTrackedAlertsRecovered = Object.fromEntries( @@ -416,16 +437,32 @@ export const createLifecycleExecutor = event[ALERT_STATUS] === ALERT_STATUS_RECOVERED && (flapping || flappingHistory.filter((f: boolean) => f).length > 0) ) - .map(({ event, flappingHistory, flapping: isCurrentlyFlapping, pendingRecoveredCount }) => { - const alertId = event[ALERT_INSTANCE_ID]!; - const alertUuid = event[ALERT_UUID]!; - const started = new Date(event[ALERT_START]!).toISOString(); - const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); - return [ - alertId, - { alertId, alertUuid, started, flappingHistory, flapping, pendingRecoveredCount }, - ]; - }) + .map( + ({ + event, + flappingHistory, + flapping: isCurrentlyFlapping, + pendingRecoveredCount, + activeCount, + }) => { + const alertId = event[ALERT_INSTANCE_ID]!; + const alertUuid = event[ALERT_UUID]!; + const started = new Date(event[ALERT_START]!).toISOString(); + const flapping = isFlapping(flappingSettings, flappingHistory, isCurrentlyFlapping); + return [ + alertId, + { + alertId, + alertUuid, + started, + flappingHistory, + flapping, + pendingRecoveredCount, + activeCount, + }, + ]; + } + ) ); return { diff --git a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts index b3047303bcb08f..abb9ebba6d016a 100644 --- a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.test.ts @@ -14,12 +14,17 @@ import { cloneDeep } from 'lodash'; import { getAlertsForNotification } from './get_alerts_for_notification'; describe('getAlertsForNotification', () => { + const newEventParams = { + maintenanceWindowIds: ['maintenance-window-id'], + timestamp: 'timestamp', + }; const alert1 = { event: { 'kibana.alert.status': ALERT_STATUS_RECOVERED, }, flapping: true, pendingRecoveredCount: 3, + activeCount: 3, }; const alert2 = { event: { @@ -40,13 +45,38 @@ describe('getAlertsForNotification', () => { pendingRecoveredCount: 4, flappingHistory: [true, true], }; + const alert5 = { + event: { + 'kibana.alert.status': ALERT_STATUS_ACTIVE, + }, + activeCount: 1, + pendingRecoveredCount: 0, + flappingHistory: [], + }; test('should set pendingRecoveredCount to zero for all active alerts', () => { - const trackedEvents = [alert4]; - expect(getAlertsForNotification(DEFAULT_FLAPPING_SETTINGS, trackedEvents)) - .toMatchInlineSnapshot(` + const trackedEvents = cloneDeep([alert4]); + const newEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification( + DEFAULT_FLAPPING_SETTINGS, + 0, + trackedEvents, + newEvents, + newEventParams + ) + ).toMatchInlineSnapshot(` Array [ Object { + "activeCount": 2, + "event": Object { + "kibana.alert.status": "active", + }, + "flappingHistory": Array [], + "pendingRecoveredCount": 0, + }, + Object { + "activeCount": 1, "event": Object { "kibana.alert.status": "active", }, @@ -62,10 +92,12 @@ describe('getAlertsForNotification', () => { test('should not remove alerts if the num of recovered alerts is not at the limit', () => { const trackedEvents = cloneDeep([alert1, alert2, alert3]); - expect(getAlertsForNotification(DEFAULT_FLAPPING_SETTINGS, trackedEvents)) - .toMatchInlineSnapshot(` + expect( + getAlertsForNotification(DEFAULT_FLAPPING_SETTINGS, 0, trackedEvents, [], newEventParams) + ).toMatchInlineSnapshot(` Array [ Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, @@ -73,12 +105,14 @@ describe('getAlertsForNotification', () => { "pendingRecoveredCount": 0, }, Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, "flapping": false, }, Object { + "activeCount": 0, "event": Object { "event.action": "active", "kibana.alert.status": "active", @@ -92,10 +126,12 @@ describe('getAlertsForNotification', () => { test('should reset counts and not modify alerts if flapping is disabled', () => { const trackedEvents = cloneDeep([alert1, alert2, alert3]); - expect(getAlertsForNotification(DISABLE_FLAPPING_SETTINGS, trackedEvents)) - .toMatchInlineSnapshot(` + expect( + getAlertsForNotification(DISABLE_FLAPPING_SETTINGS, 0, trackedEvents, [], newEventParams) + ).toMatchInlineSnapshot(` Array [ Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, @@ -103,6 +139,7 @@ describe('getAlertsForNotification', () => { "pendingRecoveredCount": 0, }, Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, @@ -110,6 +147,7 @@ describe('getAlertsForNotification', () => { "pendingRecoveredCount": 0, }, Object { + "activeCount": 0, "event": Object { "kibana.alert.status": "recovered", }, @@ -119,4 +157,106 @@ describe('getAlertsForNotification', () => { ] `); }); + + test('should increment activeCount for all active alerts', () => { + const trackedEvents = cloneDeep([alert4]); + const newEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification( + DEFAULT_FLAPPING_SETTINGS, + 0, + trackedEvents, + newEvents, + newEventParams + ) + ).toMatchInlineSnapshot(` + Array [ + Object { + "activeCount": 2, + "event": Object { + "kibana.alert.status": "active", + }, + "flappingHistory": Array [], + "pendingRecoveredCount": 0, + }, + Object { + "activeCount": 1, + "event": Object { + "kibana.alert.status": "active", + }, + "flappingHistory": Array [ + true, + true, + ], + "pendingRecoveredCount": 0, + }, + ] + `); + }); + + test('should reset activeCount for all recovered alerts', () => { + const trackedEvents = cloneDeep([alert1, alert2]); + expect( + getAlertsForNotification(DEFAULT_FLAPPING_SETTINGS, 0, trackedEvents, [], newEventParams) + ).toMatchInlineSnapshot(` + Array [ + Object { + "activeCount": 0, + "event": Object { + "kibana.alert.status": "recovered", + }, + "flapping": true, + "pendingRecoveredCount": 0, + }, + Object { + "activeCount": 0, + "event": Object { + "kibana.alert.status": "recovered", + }, + "flapping": false, + }, + ] + `); + }); + + test('should not return active alerts if the activeCount is less than the rule alertDelay', () => { + const trackedEvents = cloneDeep([alert4]); + const newEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification( + DEFAULT_FLAPPING_SETTINGS, + 5, + trackedEvents, + newEvents, + newEventParams + ) + ).toMatchInlineSnapshot(`Array []`); + }); + + test('should update active alert to look like a new alert if the activeCount is equal to the rule alertDelay', () => { + const trackedEvents = cloneDeep([alert5]); + expect( + getAlertsForNotification(DEFAULT_FLAPPING_SETTINGS, 2, trackedEvents, [], newEventParams) + ).toMatchInlineSnapshot(` + Array [ + Object { + "activeCount": 2, + "event": Object { + "event.action": "open", + "kibana.alert.duration.us": 0, + "kibana.alert.maintenance_window_ids": Array [ + "maintenance-window-id", + ], + "kibana.alert.start": "timestamp", + "kibana.alert.status": "active", + "kibana.alert.time_range": Object { + "gte": "timestamp", + }, + }, + "flappingHistory": Array [], + "pendingRecoveredCount": 0, + }, + ] + `); + }); }); diff --git a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts index 878db2a9180227..5ec0e5b835eec3 100644 --- a/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts +++ b/x-pack/plugins/rule_registry/server/utils/get_alerts_for_notification.ts @@ -11,32 +11,68 @@ import { ALERT_STATUS, ALERT_STATUS_ACTIVE, ALERT_STATUS_RECOVERED, + ALERT_START, + ALERT_DURATION, EVENT_ACTION, + ALERT_TIME_RANGE, + ALERT_MAINTENANCE_WINDOW_IDS, } from '@kbn/rule-data-utils'; export function getAlertsForNotification( flappingSettings: RulesSettingsFlappingProperties, - trackedEventsToIndex: any[] + alertDelay: number, + trackedEventsToIndex: any[], + newEventsToIndex: any[], + newEventParams: { + // values used to create a new event + maintenanceWindowIds?: string[]; + timestamp: string; + } ) { - return trackedEventsToIndex.map((trackedEvent) => { - if (!flappingSettings.enabled || trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_ACTIVE) { + const events: any[] = []; + for (const trackedEvent of [...newEventsToIndex, ...trackedEventsToIndex]) { + if (trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_ACTIVE) { + const count = trackedEvent.activeCount || 0; + trackedEvent.activeCount = count + 1; trackedEvent.pendingRecoveredCount = 0; - } else if ( - flappingSettings.enabled && - trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_RECOVERED - ) { - if (trackedEvent.flapping) { - const count = trackedEvent.pendingRecoveredCount || 0; - trackedEvent.pendingRecoveredCount = count + 1; - if (trackedEvent.pendingRecoveredCount < flappingSettings.statusChangeThreshold) { - trackedEvent.event[ALERT_STATUS] = ALERT_STATUS_ACTIVE; - trackedEvent.event[EVENT_ACTION] = 'active'; - delete trackedEvent.event[ALERT_END]; - } else { - trackedEvent.pendingRecoveredCount = 0; + // do not index the event if the number of consecutive + // active alerts is less than the rule alertDelay threshold + if (trackedEvent.activeCount < alertDelay) { + // remove from array of events to index + continue; + } else { + const { timestamp, maintenanceWindowIds } = newEventParams; + // if the active count is equal to the alertDelay it is considered a new event + if (trackedEvent.activeCount === alertDelay) { + // update the event to look like a new event + trackedEvent.event[ALERT_DURATION] = 0; + trackedEvent.event[ALERT_START] = timestamp; + trackedEvent.event[ALERT_TIME_RANGE] = { gte: timestamp }; + trackedEvent.event[EVENT_ACTION] = 'open'; + if (maintenanceWindowIds?.length) { + trackedEvent.event[ALERT_MAINTENANCE_WINDOW_IDS] = maintenanceWindowIds; + } } } + } else if (trackedEvent.event[ALERT_STATUS] === ALERT_STATUS_RECOVERED) { + trackedEvent.activeCount = 0; + if (flappingSettings.enabled) { + if (trackedEvent.flapping) { + const count = trackedEvent.pendingRecoveredCount || 0; + trackedEvent.pendingRecoveredCount = count + 1; + if (trackedEvent.pendingRecoveredCount < flappingSettings.statusChangeThreshold) { + trackedEvent.event[ALERT_STATUS] = ALERT_STATUS_ACTIVE; + trackedEvent.event[EVENT_ACTION] = 'active'; + delete trackedEvent.event[ALERT_END]; + } else { + trackedEvent.pendingRecoveredCount = 0; + } + } + } else { + trackedEvent.pendingRecoveredCount = 0; + } } - return trackedEvent; - }); + events.push(trackedEvent); + } + return events; } diff --git a/x-pack/plugins/rule_registry/server/utils/get_updated_flapping_history.test.ts b/x-pack/plugins/rule_registry/server/utils/get_updated_flapping_history.test.ts index 52467f168e641c..84685779186d97 100644 --- a/x-pack/plugins/rule_registry/server/utils/get_updated_flapping_history.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/get_updated_flapping_history.test.ts @@ -49,6 +49,7 @@ describe('getUpdatedFlappingHistory', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -81,6 +82,7 @@ describe('getUpdatedFlappingHistory', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlerts: {}, @@ -115,6 +117,7 @@ describe('getUpdatedFlappingHistory', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, trackedAlertsRecovered: {}, @@ -150,6 +153,7 @@ describe('getUpdatedFlappingHistory', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, }; @@ -184,6 +188,7 @@ describe('getUpdatedFlappingHistory', () => { flappingHistory: [], flapping: false, pendingRecoveredCount: 0, + activeCount: 0, }, }, }; diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts index 5228b1c76d3d92..da3752e098de22 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group1/event_log.ts @@ -7,6 +7,7 @@ import moment from 'moment'; import expect from '@kbn/expect'; +import { get } from 'lodash'; import { IValidatedEvent, nanosToMillis } from '@kbn/event-log-plugin/server'; import { RuleNotifyWhen } from '@kbn/alerting-plugin/common'; import { ES_TEST_INDEX_NAME, ESTestIndexTool } from '@kbn/alerting-api-integration-helpers'; @@ -1849,7 +1850,12 @@ export default function eventLogTests({ getService }: FtrProviderContext) { expect(hasActions).eql(false); }); - it('should generate expected events with a notificationDelay', async () => { + it('should generate expected events with a alertDelay', async () => { + const ACTIVE_PATH = 'kibana.alert.rule.execution.metrics.alert_counts.active'; + const NEW_PATH = 'kibana.alert.rule.execution.metrics.alert_counts.new'; + const RECOVERED_PATH = 'kibana.alert.rule.execution.metrics.alert_counts.recovered'; + const ACTION_PATH = 'kibana.alert.rule.execution.metrics.number_of_triggered_actions'; + const { body: createdAction } = await supertest .post(`${getUrlPrefix(space.id)}/api/actions/connector`) .set('kbn-xsrf', 'foo') @@ -1863,7 +1869,7 @@ export default function eventLogTests({ getService }: FtrProviderContext) { // pattern of when the alert should fire const pattern = { - instance: [true, true, true, false, true], + instance: [true, true, true, true, false, true], }; const response = await supertest @@ -1874,6 +1880,7 @@ export default function eventLogTests({ getService }: FtrProviderContext) { rule_type_id: 'test.patternFiring', schedule: { interval: '1s' }, throttle: null, + notify_when: null, params: { pattern, }, @@ -1882,9 +1889,14 @@ export default function eventLogTests({ getService }: FtrProviderContext) { id: createdAction.id, group: 'default', params: {}, + frequency: { + summary: false, + throttle: null, + notify_when: RuleNotifyWhen.CHANGE, + }, }, ], - notification_delay: { + alert_delay: { active: 3, }, }) @@ -1904,101 +1916,48 @@ export default function eventLogTests({ getService }: FtrProviderContext) { provider: 'alerting', actions: new Map([ // make sure the counts of the # of events per type are as expected - ['execute-start', { gte: 5 }], - ['execute', { gte: 5 }], - ['new-instance', { equal: 2 }], - ['active-instance', { gte: 1 }], + ['execute-start', { equal: 6 }], + ['execute', { equal: 6 }], + ['new-instance', { equal: 1 }], + ['active-instance', { equal: 2 }], ['recovered-instance', { equal: 1 }], ]), }); }); - const actualTriggeredActions = events - .filter((event) => event?.event?.action === 'execute') - .reduce( - (acc, event) => - acc + - (event?.kibana?.alert?.rule?.execution?.metrics - ?.number_of_triggered_actions as number), - 0 - ); - expect(actualTriggeredActions).to.eql(1); - }); - - it('should generate expected events with a notificationDelay with AAD', async () => { - const { body: createdAction } = await supertest - .post(`${getUrlPrefix(space.id)}/api/actions/connector`) - .set('kbn-xsrf', 'foo') - .send({ - name: 'MY action', - connector_type_id: 'test.noop', - config: {}, - secrets: {}, - }) - .expect(200); - - // pattern of when the alert should fire - const pattern = { - instance: [true, true, true, false, true], - }; - - const response = await supertest - .post(`${getUrlPrefix(space.id)}/api/alerting/rule`) - .set('kbn-xsrf', 'foo') - .send( - getTestRuleData({ - rule_type_id: 'test.patternFiringAad', - schedule: { interval: '1s' }, - throttle: null, - params: { - pattern, - }, - actions: [ - { - id: createdAction.id, - group: 'default', - params: {}, - }, - ], - notification_delay: { - active: 3, - }, - }) - ); - - expect(response.status).to.eql(200); - const alertId = response.body.id; - objectRemover.add(space.id, alertId, 'rule', 'alerting'); + const executeEvents = events.filter((event) => event?.event?.action === 'execute'); - // get the events we're expecting - const events = await retry.try(async () => { - return await getEventLog({ - getService, - spaceId: space.id, - type: 'alert', - id: alertId, - provider: 'alerting', - actions: new Map([ - // make sure the counts of the # of events per type are as expected - ['execute-start', { gte: 5 }], - ['execute', { gte: 5 }], - ['new-instance', { equal: 2 }], - ['active-instance', { gte: 1 }], - ['recovered-instance', { equal: 1 }], - ]), - }); + // first two executions do not create the active alert + executeEvents.slice(0, 1).forEach((event) => { + expect(get(event, ACTIVE_PATH)).to.be(0); + expect(get(event, NEW_PATH)).to.be(0); + expect(get(event, RECOVERED_PATH)).to.be(0); + expect(get(event, ACTION_PATH)).to.be(0); }); - const actualTriggeredActions = events - .filter((event) => event?.event?.action === 'execute') - .reduce( - (acc, event) => - acc + - (event?.kibana?.alert?.rule?.execution?.metrics - ?.number_of_triggered_actions as number), - 0 - ); - expect(actualTriggeredActions).to.eql(1); + // third executions creates the delayed active alert and triggers actions + expect(get(executeEvents[2], ACTIVE_PATH)).to.be(1); + expect(get(executeEvents[2], NEW_PATH)).to.be(1); + expect(get(executeEvents[2], RECOVERED_PATH)).to.be(0); + expect(get(executeEvents[2], ACTION_PATH)).to.be(1); + + // fourth execution + expect(get(executeEvents[3], ACTIVE_PATH)).to.be(1); + expect(get(executeEvents[3], NEW_PATH)).to.be(0); + expect(get(executeEvents[3], RECOVERED_PATH)).to.be(0); + expect(get(executeEvents[3], ACTION_PATH)).to.be(0); + + // fifth recovered execution + expect(get(executeEvents[4], ACTIVE_PATH)).to.be(0); + expect(get(executeEvents[4], NEW_PATH)).to.be(0); + expect(get(executeEvents[4], RECOVERED_PATH)).to.be(1); + expect(get(executeEvents[4], ACTION_PATH)).to.be(0); + + // sixth execution does not create the active alert + expect(get(executeEvents[5], ACTIVE_PATH)).to.be(0); + expect(get(executeEvents[5], NEW_PATH)).to.be(0); + expect(get(executeEvents[5], RECOVERED_PATH)).to.be(0); + expect(get(executeEvents[5], ACTION_PATH)).to.be(0); }); }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/notification_delay.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alert_delay.ts similarity index 82% rename from x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/notification_delay.ts rename to x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alert_delay.ts index 2b632686d57936..7062c1c65fd9c5 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/notification_delay.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alert_delay.ts @@ -12,7 +12,7 @@ import { getUrlPrefix, getTestRuleData, ObjectRemover } from '../../../../common import { Spaces } from '../../../scenarios'; // eslint-disable-next-line import/no-default-export -export default function createNotificationDelayTests({ getService }: FtrProviderContext) { +export default function createAlertDelayTests({ getService }: FtrProviderContext) { const es = getService('es'); const supertestWithoutAuth = getService('supertestWithoutAuth'); const retry = getService('retry'); @@ -22,7 +22,7 @@ export default function createNotificationDelayTests({ getService }: FtrProvider const ACTIVE_PATH = 'alertInstances.instance.meta.activeCount'; const RECOVERED_PATH = 'alertRecoveredInstances.instance.meta.activeCount'; - describe('Notification Delay', () => { + describe('Alert Delay', () => { let actionId: string; const objectRemover = new ObjectRemover(supertestWithoutAuth); @@ -37,20 +37,7 @@ export default function createNotificationDelayTests({ getService }: FtrProvider afterEach(() => objectRemover.removeAll()); - it('should clear the activeCount if the notificationDelay is not configured for the rule', async () => { - const start = new Date().toISOString(); - const pattern = { - instance: [true], - }; - - const ruleId = await createRule(actionId, pattern); - objectRemover.add(space.id, ruleId, 'rule', 'alerting'); - - const state = await getAlertState(start, ruleId, 0); - expect(get(state, ACTIVE_PATH)).to.eql(0); - }); - - it('should update the activeCount when alert is active and clear when recovered if the notificationDelay is configured for the rule', async () => { + it('should update the activeCount when alert is active and clear when recovered', async () => { let start = new Date().toISOString(); const pattern = { instance: [true, true, true, false, true], @@ -79,7 +66,7 @@ export default function createNotificationDelayTests({ getService }: FtrProvider expect(get(state, ACTIVE_PATH)).to.eql(1); }); - it('should reset the activeCount when count of consecutive active alerts exceeds the notificationDelay count', async () => { + it('should continue incrementing the activeCount when count of consecutive active alerts exceeds the alertDelay count', async () => { let start = new Date().toISOString(); const pattern = { instance: [true, true, true, true, true], @@ -96,16 +83,16 @@ export default function createNotificationDelayTests({ getService }: FtrProvider expect(get(state, ACTIVE_PATH)).to.eql(2); start = new Date().toISOString(); - state = await getAlertState(start, ruleId, 0, true); - expect(get(state, ACTIVE_PATH)).to.eql(0); + state = await getAlertState(start, ruleId, 3, true); + expect(get(state, ACTIVE_PATH)).to.eql(3); start = new Date().toISOString(); - state = await getAlertState(start, ruleId, 1, true); - expect(get(state, ACTIVE_PATH)).to.eql(1); + state = await getAlertState(start, ruleId, 4, true); + expect(get(state, ACTIVE_PATH)).to.eql(4); start = new Date().toISOString(); - state = await getAlertState(start, ruleId, 2, true); - expect(get(state, ACTIVE_PATH)).to.eql(2); + state = await getAlertState(start, ruleId, 5, true); + expect(get(state, ACTIVE_PATH)).to.eql(5); }); }); @@ -187,7 +174,7 @@ export default function createNotificationDelayTests({ getService }: FtrProvider params: {}, }, ], - ...(activeCount ? { notification_delay: { active: activeCount } } : {}), + ...(activeCount ? { alert_delay: { active: activeCount } } : {}), }) ) .expect(200); diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/alerts_as_data_alert_delay.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/alerts_as_data_alert_delay.ts new file mode 100644 index 00000000000000..f7e2876e9775b2 --- /dev/null +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/alerts_as_data_alert_delay.ts @@ -0,0 +1,665 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { get } from 'lodash'; +import { SearchHit } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { IValidatedEvent } from '@kbn/event-log-plugin/server'; +import type { Alert } from '@kbn/alerts-as-data-utils'; +import { + ALERT_ACTION_GROUP, + ALERT_DURATION, + ALERT_END, + ALERT_RULE_CATEGORY, + ALERT_RULE_CONSUMER, + ALERT_RULE_EXECUTION_UUID, + ALERT_RULE_NAME, + ALERT_RULE_PARAMETERS, + ALERT_RULE_PRODUCER, + ALERT_RULE_TAGS, + ALERT_RULE_TYPE_ID, + ALERT_RULE_UUID, + ALERT_START, + ALERT_STATUS, + ALERT_TIME_RANGE, + ALERT_UUID, + ALERT_WORKFLOW_STATUS, + EVENT_ACTION, + EVENT_KIND, + SPACE_IDS, +} from '@kbn/rule-data-utils'; +import { RuleNotifyWhen } from '@kbn/alerting-plugin/common'; +import { ES_TEST_INDEX_NAME, ESTestIndexTool } from '@kbn/alerting-api-integration-helpers'; +import { FtrProviderContext } from '../../../../../common/ftr_provider_context'; +import { Spaces } from '../../../../scenarios'; +import { + getEventLog, + getTestRuleData, + getUrlPrefix, + ObjectRemover, + TaskManagerDoc, +} from '../../../../../common/lib'; + +// eslint-disable-next-line import/no-default-export +export default function createAlertsAsDataAlertDelayInstallResourcesTest({ + getService, +}: FtrProviderContext) { + const ACTIVE_PATH = 'kibana.alert.rule.execution.metrics.alert_counts.active'; + const NEW_PATH = 'kibana.alert.rule.execution.metrics.alert_counts.new'; + const RECOVERED_PATH = 'kibana.alert.rule.execution.metrics.alert_counts.recovered'; + const ACTION_PATH = 'kibana.alert.rule.execution.metrics.number_of_triggered_actions'; + const UUID_PATH = 'kibana.alert.rule.execution.uuid'; + + const es = getService('es'); + const retry = getService('retry'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + const objectRemover = new ObjectRemover(supertestWithoutAuth); + const esTestIndexTool = new ESTestIndexTool(es, retry); + + type PatternFiringAlert = Alert & { patternIndex: number; instancePattern: boolean[] }; + // type AlwaysFiringAlert = Alert & { patternIndex: number; instancePattern: boolean[] }; + + const alertsAsDataIndex = '.alerts-test.patternfiring.alerts-default'; + const alwaysFiringAlertsAsDataIndex = + '.internal.alerts-observability.test.alerts.alerts-default-000001'; + const timestampPattern = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/; + + describe('alerts as data', () => { + before(async () => { + await esTestIndexTool.destroy(); + await esTestIndexTool.setup(); + await es.deleteByQuery({ + index: [alertsAsDataIndex, alwaysFiringAlertsAsDataIndex], + query: { match_all: {} }, + conflicts: 'proceed', + }); + }); + afterEach(() => objectRemover.removeAll()); + after(async () => { + await objectRemover.removeAll(); + await esTestIndexTool.destroy(); + await es.deleteByQuery({ + index: [alertsAsDataIndex, alwaysFiringAlertsAsDataIndex], + query: { match_all: {} }, + conflicts: 'proceed', + }); + }); + + it('should generate expected events with a alertDelay with AAD', async () => { + const { body: createdAction } = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .send({ + name: 'MY action', + connector_type_id: 'test.noop', + config: {}, + secrets: {}, + }) + .expect(200); + + // pattern of when the alert should fire + const pattern = { + instance: [true, true, true, true, false, true], + }; + + const response = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.patternFiringAad', + schedule: { interval: '1d' }, + throttle: null, + notify_when: null, + params: { + pattern, + }, + actions: [ + { + id: createdAction.id, + group: 'default', + params: {}, + frequency: { + summary: false, + throttle: null, + notify_when: RuleNotifyWhen.CHANGE, + }, + }, + ], + alert_delay: { + active: 3, + }, + }) + ); + + expect(response.status).to.eql(200); + const ruleId = response.body.id; + objectRemover.add(Spaces.space1.id, ruleId, 'rule', 'alerting'); + + // -------------------------- + // RUN 1 - 0 new alerts + // -------------------------- + let events: IValidatedEvent[] = await waitForEventLogDocs( + ruleId, + new Map([['execute', { equal: 1 }]]) + ); + let executeEvent = events[0]; + expect(get(executeEvent, ACTIVE_PATH)).to.be(0); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun1 = await queryForAlertDocs(); + + // Get alert state from task document + let state: any = await getTaskState(ruleId); + expect(state.alertInstances.instance.meta.activeCount).to.equal(1); + expect(state.alertInstances.instance.state.patternIndex).to.equal(0); + + // After the first run, we should have 0 alert docs for the 0 active alerts + expect(alertDocsRun1.length).to.equal(0); + + // -------------------------- + // RUN 2 - 0 new alerts + // -------------------------- + let runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 2 }]])); + executeEvent = events[1]; + expect(get(executeEvent, ACTIVE_PATH)).to.be(0); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun2 = await queryForAlertDocs(); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertInstances.instance.meta.activeCount).to.equal(2); + expect(state.alertInstances.instance.state.patternIndex).to.equal(1); + + // After the second run, we should have 0 alert docs for the 0 active alerts + expect(alertDocsRun2.length).to.equal(0); + + // -------------------------- + // RUN 3 - 1 new alert + // -------------------------- + runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 3 }]])); + executeEvent = events[2]; + let executionUuid = get(executeEvent, UUID_PATH); + expect(get(executeEvent, ACTIVE_PATH)).to.be(1); + expect(get(executeEvent, NEW_PATH)).to.be(1); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(1); + + // Query for alerts + const alertDocsRun3 = await queryForAlertDocs(); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertInstances.instance.meta.activeCount).to.equal(3); + expect(state.alertInstances.instance.state.patternIndex).to.equal(2); + + // After the third run, we should have 1 alert docs for the 1 active alert + expect(alertDocsRun3.length).to.equal(1); + + testExpectRuleData(alertDocsRun3, ruleId, { pattern }, executionUuid!); + let source: PatternFiringAlert = alertDocsRun3[0]._source!; + + // Each doc should have active status and default action group id + expect(source[ALERT_ACTION_GROUP]).to.equal('default'); + // patternIndex should be 2 for the third run + expect(source.patternIndex).to.equal(2); + // alert UUID should equal doc id + expect(source[ALERT_UUID]).to.equal(alertDocsRun3[0]._id); + // duration should be 0 since this is a new alert + expect(source[ALERT_DURATION]).to.equal(0); + // start should be defined + expect(source[ALERT_START]).to.match(timestampPattern); + // time_range.gte should be same as start + expect(source[ALERT_TIME_RANGE]?.gte).to.equal(source[ALERT_START]); + // timestamp should be defined + expect(source['@timestamp']).to.match(timestampPattern); + // status should be active + expect(source[ALERT_STATUS]).to.equal('active'); + // workflow status should be 'open' + expect(source[ALERT_WORKFLOW_STATUS]).to.equal('open'); + // event.action should be 'open' + expect(source[EVENT_ACTION]).to.equal('open'); + // event.kind should be 'signal' + expect(source[EVENT_KIND]).to.equal('signal'); + // tags should equal rule tags because rule type doesn't set any tags + expect(source.tags).to.eql(['foo']); + + // -------------------------- + // RUN 4 - 1 active alert + // -------------------------- + runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 4 }]])); + executeEvent = events[3]; + executionUuid = get(executeEvent, UUID_PATH); + expect(get(executeEvent, ACTIVE_PATH)).to.be(1); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun4 = await queryForAlertDocs(); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertInstances.instance.meta.activeCount).to.equal(4); + expect(state.alertInstances.instance.state.patternIndex).to.equal(3); + + // After the fourth run, we should have 1 alert docs for the 1 active alert + expect(alertDocsRun4.length).to.equal(1); + + testExpectRuleData(alertDocsRun4, ruleId, { pattern }, executionUuid!); + source = alertDocsRun4[0]._source!; + const run3Source = alertDocsRun3[0]._source!; + + expect(source[ALERT_UUID]).to.equal(run3Source[ALERT_UUID]); + // patternIndex should be 3 for the fourth run + expect(source.patternIndex).to.equal(3); + expect(source[ALERT_ACTION_GROUP]).to.equal('default'); + // start time should be defined and the same as prior run + expect(source[ALERT_START]).to.match(timestampPattern); + expect(source[ALERT_START]).to.equal(run3Source[ALERT_START]); + // timestamp should be defined and not the same as prior run + expect(source['@timestamp']).to.match(timestampPattern); + expect(source['@timestamp']).not.to.equal(run3Source['@timestamp']); + // status should still be active + expect(source[ALERT_STATUS]).to.equal('active'); + // event.action set to active + expect(source[EVENT_ACTION]).to.eql('active'); + expect(source.tags).to.eql(['foo']); + // these values should be the same as previous run + expect(source[EVENT_KIND]).to.eql(run3Source[EVENT_KIND]); + expect(source[ALERT_WORKFLOW_STATUS]).to.eql(run3Source[ALERT_WORKFLOW_STATUS]); + expect(source[ALERT_TIME_RANGE]?.gte).to.equal(run3Source[ALERT_TIME_RANGE]?.gte); + + // -------------------------- + // RUN 5 - 1 recovered alert + // -------------------------- + runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 5 }]])); + executeEvent = events[4]; + executionUuid = get(executeEvent, UUID_PATH); + expect(get(executeEvent, ACTIVE_PATH)).to.be(0); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(1); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun5 = await queryForAlertDocs(); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertRecoveredInstances.instance.meta.activeCount).to.equal(0); + + // After the fourth run, we should have 1 alert docs for the 1 recovered alert + expect(alertDocsRun5.length).to.equal(1); + + testExpectRuleData(alertDocsRun5, ruleId, { pattern }, executionUuid!); + source = alertDocsRun5[0]._source!; + + // action group should be set to recovered + expect(source[ALERT_ACTION_GROUP]).to.be('recovered'); + // rule type AAD payload should be set to recovery values + expect(source.instancePattern).to.eql([]); + expect(source.patternIndex).to.eql(-1); + // uuid is the same + expect(source[ALERT_UUID]).to.equal(run3Source[ALERT_UUID]); + // start time should be defined and the same as before + expect(source[ALERT_START]).to.match(timestampPattern); + expect(source[ALERT_START]).to.equal(run3Source[ALERT_START]); + // timestamp should be defined and not the same as prior run + expect(source['@timestamp']).to.match(timestampPattern); + expect(source['@timestamp']).not.to.equal(run3Source['@timestamp']); + // end time should be defined + expect(source[ALERT_END]).to.match(timestampPattern); + // status should be set to recovered + expect(source[ALERT_STATUS]).to.equal('recovered'); + // event.action set to close + expect(source[EVENT_ACTION]).to.eql('close'); + expect(source.tags).to.eql(['foo']); + // these values should be the same as previous run + expect(source[EVENT_KIND]).to.eql(run3Source[EVENT_KIND]); + expect(source[ALERT_WORKFLOW_STATUS]).to.eql(run3Source[ALERT_WORKFLOW_STATUS]); + expect(source[ALERT_TIME_RANGE]?.gte).to.equal(run3Source[ALERT_TIME_RANGE]?.gte); + // time_range.lte should be set to end time + expect(source[ALERT_TIME_RANGE]?.lte).to.equal(source[ALERT_END]); + + // -------------------------- + // RUN 6 - 0 new alerts + // -------------------------- + runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 6 }]])); + executeEvent = events[5]; + expect(get(executeEvent, ACTIVE_PATH)).to.be(0); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun6 = await queryForAlertDocs(); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertInstances.instance.meta.activeCount).to.equal(1); + expect(state.alertInstances.instance.state.patternIndex).to.equal(5); + + // After the sixth run, we should have 1 alert docs for the previously recovered alert + expect(alertDocsRun6.length).to.equal(1); + }); + + it('should generate expected events with a alertDelay with AAD (rule registry)', async () => { + const params = { + index: ES_TEST_INDEX_NAME, + reference: 'test', + }; + const { body: createdAction } = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/api/actions/connector`) + .set('kbn-xsrf', 'foo') + .send({ + name: 'MY action', + connector_type_id: 'test.noop', + config: {}, + secrets: {}, + }) + .expect(200); + + const response = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/api/alerting/rule`) + .set('kbn-xsrf', 'foo') + .send( + getTestRuleData({ + rule_type_id: 'test.always-firing-alert-as-data', + schedule: { interval: '1d' }, + throttle: null, + notify_when: null, + params, + actions: [ + { + id: createdAction.id, + group: 'default', + params: {}, + frequency: { + summary: false, + throttle: null, + notify_when: RuleNotifyWhen.CHANGE, + }, + }, + ], + alert_delay: { + active: 3, + }, + }) + ); + + expect(response.status).to.eql(200); + const ruleId = response.body.id; + objectRemover.add(Spaces.space1.id, ruleId, 'rule', 'alerting'); + + // -------------------------- + // RUN 1 - 0 new alerts + // -------------------------- + let events: IValidatedEvent[] = await waitForEventLogDocs( + ruleId, + new Map([['execute', { equal: 1 }]]) + ); + let executeEvent = events[0]; + expect(get(executeEvent, ACTIVE_PATH)).to.be(0); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun1 = await queryForAlertDocs(alwaysFiringAlertsAsDataIndex); + + // Get alert state from task document + let state: any = await getTaskState(ruleId); + expect(state.alertInstances['1'].meta.activeCount).to.equal(1); + expect(state.alertTypeState.trackedAlerts['1'].activeCount).to.equal(1); + + // After the first run, we should have 0 alert docs for the 0 active alerts + expect(alertDocsRun1.length).to.equal(0); + + // -------------------------- + // RUN 2 - 0 new alerts + // -------------------------- + let runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 2 }]])); + executeEvent = events[1]; + expect(get(executeEvent, ACTIVE_PATH)).to.be(0); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun2 = await queryForAlertDocs(alwaysFiringAlertsAsDataIndex); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertInstances['1'].meta.activeCount).to.equal(2); + expect(state.alertTypeState.trackedAlerts['1'].activeCount).to.equal(2); + + // After the second run, we should have 0 alert docs for the 0 active alerts + expect(alertDocsRun2.length).to.equal(0); + + // -------------------------- + // RUN 3 - 1 new alert + // -------------------------- + runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 3 }]])); + executeEvent = events[2]; + let executionUuid = get(executeEvent, UUID_PATH); + // Note: the rule creates 2 alerts but we will only look at one + expect(get(executeEvent, ACTIVE_PATH)).to.be(2); + expect(get(executeEvent, NEW_PATH)).to.be(2); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(2); + + // Query for alerts + const alertDocsRun3 = await queryForAlertDocs(alwaysFiringAlertsAsDataIndex); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertInstances['1'].meta.activeCount).to.equal(3); + expect(state.alertTypeState.trackedAlerts['1'].activeCount).to.equal(3); + + // After the third run, we should have 2 alert docs for the 2 active alerts but we will only look at one + expect(alertDocsRun3.length).to.equal(2); + + let source: Alert = alertDocsRun3[0]._source!; + + // Each doc should have a copy of the rule data + expect(source[ALERT_RULE_CATEGORY]).to.equal('Test: Always Firing Alert As Data'); + expect(source[ALERT_RULE_CONSUMER]).to.equal('alertsFixture'); + expect(source[ALERT_RULE_NAME]).to.equal('abc'); + expect(source[ALERT_RULE_PRODUCER]).to.equal('alertsFixture'); + expect(source[ALERT_RULE_TAGS]).to.eql(['foo']); + expect(source[ALERT_RULE_TYPE_ID]).to.equal('test.always-firing-alert-as-data'); + expect(source[ALERT_RULE_UUID]).to.equal(ruleId); + expect(source[ALERT_RULE_PARAMETERS]).to.eql(params); + expect(source[SPACE_IDS]).to.eql(['space1']); + expect(source[ALERT_RULE_EXECUTION_UUID]).to.equal(executionUuid); + // alert UUID should equal doc id + expect(source[ALERT_UUID]).to.equal(alertDocsRun3[0]._id); + // duration should be 0 since this is a new alert + expect(source[ALERT_DURATION]).to.equal(0); + // start should be defined + expect(source[ALERT_START]).to.match(timestampPattern); + // time_range.gte should be same as start + expect(source[ALERT_TIME_RANGE]?.gte).to.equal(source[ALERT_START]); + // timestamp should be defined + expect(source['@timestamp']).to.match(timestampPattern); + // status should be active + expect(source[ALERT_STATUS]).to.equal('active'); + // workflow status should be 'open' + expect(source[ALERT_WORKFLOW_STATUS]).to.equal('open'); + // event.action should be 'open' + expect(source[EVENT_ACTION]).to.equal('open'); + // event.kind should be 'signal' + expect(source[EVENT_KIND]).to.equal('signal'); + // tags should equal rule tags because rule type doesn't set any tags + expect(source.tags).to.eql(['foo']); + + // -------------------------- + // RUN 4 - 1 active alert + // -------------------------- + runSoon = await supertestWithoutAuth + .post(`${getUrlPrefix(Spaces.space1.id)}/internal/alerting/rule/${ruleId}/_run_soon`) + .set('kbn-xsrf', 'foo'); + expect(runSoon.status).to.eql(204); + + events = await waitForEventLogDocs(ruleId, new Map([['execute', { equal: 4 }]])); + executeEvent = events[3]; + executionUuid = get(executeEvent, UUID_PATH); + // Note: the rule creates 2 alerts but we will only look at one + expect(get(executeEvent, ACTIVE_PATH)).to.be(2); + expect(get(executeEvent, NEW_PATH)).to.be(0); + expect(get(executeEvent, RECOVERED_PATH)).to.be(0); + expect(get(executeEvent, ACTION_PATH)).to.be(0); + + // Query for alerts + const alertDocsRun4 = await queryForAlertDocs(alwaysFiringAlertsAsDataIndex); + + // Get alert state from task document + state = await getTaskState(ruleId); + expect(state.alertInstances['1'].meta.activeCount).to.equal(4); + expect(state.alertTypeState.trackedAlerts['1'].activeCount).to.equal(4); + + // After the fourth run, we should have 2 alert docs for the 2 active alerts but we will only look at one + expect(alertDocsRun4.length).to.equal(2); + + source = alertDocsRun4[0]._source!; + const run3Source = alertDocsRun3[0]._source!; + + // Each doc should have a copy of the rule data + expect(source[ALERT_RULE_CATEGORY]).to.equal('Test: Always Firing Alert As Data'); + expect(source[ALERT_RULE_CONSUMER]).to.equal('alertsFixture'); + expect(source[ALERT_RULE_NAME]).to.equal('abc'); + expect(source[ALERT_RULE_PRODUCER]).to.equal('alertsFixture'); + expect(source[ALERT_RULE_TAGS]).to.eql(['foo']); + expect(source[ALERT_RULE_TYPE_ID]).to.equal('test.always-firing-alert-as-data'); + expect(source[ALERT_RULE_UUID]).to.equal(ruleId); + expect(source[ALERT_RULE_PARAMETERS]).to.eql(params); + expect(source[SPACE_IDS]).to.eql(['space1']); + expect(source[ALERT_RULE_EXECUTION_UUID]).to.equal(executionUuid); + expect(source[ALERT_UUID]).to.equal(run3Source[ALERT_UUID]); + // start time should be defined and the same as prior run + expect(source[ALERT_START]).to.match(timestampPattern); + expect(source[ALERT_START]).to.equal(run3Source[ALERT_START]); + // timestamp should be defined and not the same as prior run + expect(source['@timestamp']).to.match(timestampPattern); + expect(source['@timestamp']).not.to.equal(run3Source['@timestamp']); + // status should still be active + expect(source[ALERT_STATUS]).to.equal('active'); + // event.action set to active + expect(source[EVENT_ACTION]).to.eql('active'); + expect(source.tags).to.eql(['foo']); + // these values should be the same as previous run + expect(source[EVENT_KIND]).to.eql(run3Source[EVENT_KIND]); + expect(source[ALERT_WORKFLOW_STATUS]).to.eql(run3Source[ALERT_WORKFLOW_STATUS]); + expect(source[ALERT_TIME_RANGE]?.gte).to.equal(run3Source[ALERT_TIME_RANGE]?.gte); + }); + }); + + function testExpectRuleData( + alertDocs: Array>, + ruleId: string, + ruleParameters: unknown, + executionUuid?: string + ) { + for (let i = 0; i < alertDocs.length; ++i) { + const source: PatternFiringAlert = alertDocs[i]._source!; + + // Each doc should have a copy of the rule data + expect(source[ALERT_RULE_CATEGORY]).to.equal( + 'Test: Firing on a Pattern and writing Alerts as Data' + ); + expect(source[ALERT_RULE_CONSUMER]).to.equal('alertsFixture'); + expect(source[ALERT_RULE_NAME]).to.equal('abc'); + expect(source[ALERT_RULE_PRODUCER]).to.equal('alertsFixture'); + expect(source[ALERT_RULE_TAGS]).to.eql(['foo']); + expect(source[ALERT_RULE_TYPE_ID]).to.equal('test.patternFiringAad'); + expect(source[ALERT_RULE_UUID]).to.equal(ruleId); + expect(source[ALERT_RULE_PARAMETERS]).to.eql(ruleParameters); + expect(source[SPACE_IDS]).to.eql(['space1']); + + if (executionUuid) { + expect(source[ALERT_RULE_EXECUTION_UUID]).to.equal(executionUuid); + } + } + } + + async function queryForAlertDocs( + index: string = alertsAsDataIndex + ): Promise>> { + const searchResult = await es.search({ + index, + body: { query: { match_all: {} } }, + }); + return searchResult.hits.hits as Array>; + } + + async function getTaskState(ruleId: string) { + const task = await es.get({ + id: `task:${ruleId}`, + index: '.kibana_task_manager', + }); + + return JSON.parse(task._source!.task.state); + } + + async function waitForEventLogDocs( + id: string, + actions: Map + ) { + return await retry.try(async () => { + return await getEventLog({ + getService, + spaceId: Spaces.space1.id, + type: 'alert', + id, + provider: 'alerting', + actions, + }); + }); + } +} diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts index 20342e053016dc..e1a29d1c4bf3e6 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data/index.ts @@ -14,5 +14,6 @@ export default function alertsAsDataTests({ loadTestFile }: FtrProviderContext) loadTestFile(require.resolve('./alerts_as_data')); loadTestFile(require.resolve('./alerts_as_data_flapping')); loadTestFile(require.resolve('./alerts_as_data_conflicts')); + loadTestFile(require.resolve('./alerts_as_data_alert_delay')); }); } diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/index.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/index.ts index b73477cf3df302..15084a47f4d863 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/index.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/index.ts @@ -28,7 +28,7 @@ export default function alertingTests({ loadTestFile, getService }: FtrProviderC loadTestFile(require.resolve('./run_soon')); loadTestFile(require.resolve('./flapping_history')); loadTestFile(require.resolve('./check_registered_rule_types')); - loadTestFile(require.resolve('./notification_delay')); + loadTestFile(require.resolve('./alert_delay')); loadTestFile(require.resolve('./generate_alert_schemas')); // Do not place test files here, due to https://github.com/elastic/kibana/issues/123059