Updating the alert creation delay

elastic · Jan 29, 2024 · 9f3575a · 9f3575a
1 parent 8aec35a
commit 9f3575a
Show file tree

Hide file tree

Showing 21 changed files with 439 additions and 234 deletions.
diff --git a/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts b/x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts
@@ -20,6 +20,8 @@ const trackedAlertStateRt = t.type({
   // count of consecutive recovered alerts for flapping
   // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
   pendingRecoveredCount: t.number,
+  // count of consecutive active alerts will reset if the alert is recovered
+  activeCount: t.number,
 });
 
 export type TrackedLifecycleAlertState = t.TypeOf<typeof trackedAlertStateRt>;

diff --git a/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts b/x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts
@@ -36,8 +36,7 @@ export const metaSchema = schema.object({
   // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
   pendingRecoveredCount: schema.maybe(schema.number()),
   uuid: schema.maybe(schema.string()),
-  // count of consecutive active alerts
-  // will reset if the alert is recovered or if equal to alertDelay.active stored in the rule
+  // count of consecutive active alerts will reset if the alert is recovered
   activeCount: schema.maybe(schema.number()),
 });
 

diff --git a/x-pack/plugins/alerting/common/rule.ts b/x-pack/plugins/alerting/common/rule.ts
@@ -222,6 +222,7 @@ export type SanitizedRuleConfig = Pick<
   | 'muteAll'
   | 'revision'
   | 'snoozeSchedule'
+  | 'alertDelay'
 > & {
   producer: string;
   ruleTypeId: string;

diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.test.ts
@@ -308,6 +308,7 @@ describe('Alerts Client', () => {
           flappingSettings: DEFAULT_FLAPPING_SETTINGS,
           notifyOnActionGroupChange: true,
           maintenanceWindowIds: [],
+          alertDelay: 0,
         };
       });
 
@@ -516,6 +517,25 @@ describe('Alerts Client', () => {
           });
         });
 
+        test('should not index new alerts if the activeCount is less than the rule alertDelay', async () => {
+          const alertsClient = new AlertsClient<{}, {}, {}, 'default', 'recovered'>({
+            ...alertsClientParams,
+            rule: { ...alertsClientParams.rule, alertDelay: 3 },
+          });
+
+          await alertsClient.initializeExecution(defaultExecutionOpts);
+
+          // Report 1 new alerts
+          const alertExecutorService = alertsClient.factory();
+          alertExecutorService.create('1').scheduleActions('default');
+
+          alertsClient.processAndLogAlerts(processAndLogAlertsOpts);
+
+          await alertsClient.persistAlerts();
+
+          expect(clusterClient.bulk).not.toHaveBeenCalled();
+        });
+
         test('should update ongoing alerts in existing index', async () => {
           clusterClient.search.mockResolvedValue({
             took: 10,

diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client.ts
@@ -343,6 +343,11 @@ export class AlertsClient<
             })
           );
         } else {
+          // skip writing the alert document if the number of consecutive
+          // active alerts is less than the rule alertDelay threshold
+          if (activeAlerts[id].getActiveCount() < this.options.rule.alertDelay) {
+            continue;
+          }
           activeAlertsToIndex.push(
             buildNewAlert<
               AlertData,

diff --git a/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts b/x-pack/plugins/alerting/server/alerts_client/alerts_client_fixtures.ts
@@ -25,6 +25,7 @@ export const alertRuleData: AlertRuleData = {
   revision: 0,
   spaceId: 'default',
   tags: ['rule-', '-tags'],
+  alertDelay: 0,
 };
 
 export const mockAAD = {

diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.test.ts
@@ -245,6 +245,7 @@ describe('Legacy Alerts Client', () => {
       flappingSettings: DEFAULT_FLAPPING_SETTINGS,
       notifyOnActionGroupChange: true,
       maintenanceWindowIds: ['window-id1', 'window-id2'],
+      alertDelay: 5,
     });
 
     expect(processAlerts).toHaveBeenCalledWith({
@@ -275,13 +276,15 @@ describe('Legacy Alerts Client', () => {
       },
       true,
       'default',
+      5,
       {},
       {
         '1': new Alert<AlertInstanceContext, AlertInstanceContext>('1', testAlert1),
         '2': new Alert<AlertInstanceContext, AlertInstanceContext>('2', testAlert2),
       },
       {},
-      {}
+      {},
+      null
     );
 
     expect(logAlerts).toHaveBeenCalledWith({

diff --git a/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts b/x-pack/plugins/alerting/server/alerts_client/legacy_alerts_client.ts
@@ -140,6 +140,7 @@ export class LegacyAlertsClient<
     notifyOnActionGroupChange,
     flappingSettings,
     maintenanceWindowIds,
+    alertDelay,
   }: ProcessAlertsOpts) {
     const {
       newAlerts: processedAlertsNew,
@@ -168,10 +169,12 @@ export class LegacyAlertsClient<
       flappingSettings,
       notifyOnActionGroupChange,
       this.options.ruleType.defaultActionGroupId,
+      alertDelay,
       processedAlertsNew,
       processedAlertsActive,
       trimmedAlertsRecovered,
-      processedAlertsRecoveredCurrent
+      processedAlertsRecoveredCurrent,
+      this.startedAtString
     );
     alerts.currentRecoveredAlerts = merge(alerts.currentRecoveredAlerts, earlyRecoveredAlerts);
 
@@ -203,11 +206,13 @@ export class LegacyAlertsClient<
     flappingSettings,
     notifyOnActionGroupChange,
     maintenanceWindowIds,
+    alertDelay,
   }: ProcessAndLogAlertsOpts) {
     this.processAlerts({
       notifyOnActionGroupChange,
       flappingSettings,
       maintenanceWindowIds,
+      alertDelay,
     });
 
     this.logAlerts({

diff --git a/x-pack/plugins/alerting/server/alerts_client/types.ts b/x-pack/plugins/alerting/server/alerts_client/types.ts
@@ -47,6 +47,7 @@ export interface AlertRuleData {
   revision: number;
   spaceId: string;
   tags: string[];
+  alertDelay: number;
 }
 
 export interface AlertRule {
@@ -111,12 +112,14 @@ export interface ProcessAndLogAlertsOpts {
   flappingSettings: RulesSettingsFlappingProperties;
   notifyOnActionGroupChange: boolean;
   maintenanceWindowIds: string[];
+  alertDelay: number;
 }
 
 export interface ProcessAlertsOpts {
   flappingSettings: RulesSettingsFlappingProperties;
   notifyOnActionGroupChange: boolean;
   maintenanceWindowIds: string[];
+  alertDelay: number;
 }
 
 export interface LogAlertsOpts {

diff --git a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.test.ts
@@ -21,6 +21,7 @@ describe('getAlertsForNotification', () => {
       DEFAULT_FLAPPING_SETTINGS,
       true,
       'default',
+      0,
       {
         '1': alert1,
       },
@@ -89,6 +90,7 @@ describe('getAlertsForNotification', () => {
       DEFAULT_FLAPPING_SETTINGS,
       true,
       'default',
+      0,
       {},
       {},
       {
@@ -222,6 +224,7 @@ describe('getAlertsForNotification', () => {
         DISABLE_FLAPPING_SETTINGS,
         true,
         'default',
+        0,
         {},
         {},
         {
@@ -353,6 +356,7 @@ describe('getAlertsForNotification', () => {
       DEFAULT_FLAPPING_SETTINGS,
       false,
       'default',
+      0,
       {},
       {},
       {
@@ -455,10 +459,11 @@ describe('getAlertsForNotification', () => {
     });
     const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } });
 
-    const { newAlerts, activeAlerts } = getAlertsForNotification(
+    const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification(
       DEFAULT_FLAPPING_SETTINGS,
       true,
       'default',
+      0,
       {
         '1': alert1,
       },
@@ -507,6 +512,30 @@ describe('getAlertsForNotification', () => {
         },
       }
     `);
+    expect(currentActiveAlerts).toMatchInlineSnapshot(`
+      Object {
+        "1": Object {
+          "meta": Object {
+            "activeCount": 2,
+            "flappingHistory": Array [],
+            "maintenanceWindowIds": Array [],
+            "pendingRecoveredCount": 0,
+            "uuid": "uuid-1",
+          },
+          "state": Object {},
+        },
+        "2": Object {
+          "meta": Object {
+            "activeCount": 1,
+            "flappingHistory": Array [],
+            "maintenanceWindowIds": Array [],
+            "pendingRecoveredCount": 0,
+            "uuid": "uuid-2",
+          },
+          "state": Object {},
+        },
+      }
+    `);
   });
 
   test('should reset activeCount for all recovered alerts', () => {
@@ -517,6 +546,7 @@ describe('getAlertsForNotification', () => {
       DEFAULT_FLAPPING_SETTINGS,
       true,
       'default',
+      0,
       {},
       {},
       {
@@ -574,4 +604,78 @@ describe('getAlertsForNotification', () => {
       }
     `);
   });
+
+  test('should remove the alert from newAlerts and should not return the alert in currentActiveAlerts if the activeCount is less than the rule alertDelay', () => {
+    const alert1 = new Alert('1', {
+      meta: { activeCount: 1, uuid: 'uuid-1' },
+    });
+    const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } });
+
+    const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification(
+      DEFAULT_FLAPPING_SETTINGS,
+      true,
+      'default',
+      5,
+      {
+        '1': alert1,
+      },
+      {
+        '1': alert1,
+        '2': alert2,
+      },
+      {},
+      {}
+    );
+    expect(newAlerts).toMatchInlineSnapshot(`Object {}`);
+    expect(activeAlerts).toMatchInlineSnapshot(`
+      Object {
+        "1": Object {
+          "meta": Object {
+            "activeCount": 2,
+            "flappingHistory": Array [],
+            "maintenanceWindowIds": Array [],
+            "pendingRecoveredCount": 0,
+            "uuid": "uuid-1",
+          },
+          "state": Object {},
+        },
+        "2": Object {
+          "meta": Object {
+            "activeCount": 1,
+            "flappingHistory": Array [],
+            "maintenanceWindowIds": Array [],
+            "pendingRecoveredCount": 0,
+            "uuid": "uuid-2",
+          },
+          "state": Object {},
+        },
+      }
+    `);
+    expect(currentActiveAlerts).toMatchInlineSnapshot(`Object {}`);
+  });
+
+  test('should update active alert to look like a new alert if the activeCount is equal to the rule alertDelay', () => {
+    const alert2 = new Alert('2', { meta: { uuid: 'uuid-2' } });
+
+    const { newAlerts, activeAlerts, currentActiveAlerts } = getAlertsForNotification(
+      DEFAULT_FLAPPING_SETTINGS,
+      true,
+      'default',
+      1,
+      {},
+      {
+        '2': alert2,
+      },
+      {},
+      {}
+    );
+    expect(newAlerts['2'].getState().duration).toBe('0');
+    expect(newAlerts['2'].getState().start).toBeTruthy();
+
+    expect(activeAlerts['2'].getState().duration).toBe('0');
+    expect(activeAlerts['2'].getState().start).toBeTruthy();
+
+    expect(currentActiveAlerts['2'].getState().duration).toBe('0');
+    expect(currentActiveAlerts['2'].getState().start).toBeTruthy();
+  });
 });
diff --git a/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts b/x-pack/plugins/alerting/server/lib/get_alerts_for_notification.ts
@@ -19,18 +19,35 @@ export function getAlertsForNotification<
   flappingSettings: RulesSettingsFlappingProperties,
   notifyOnActionGroupChange: boolean,
   actionGroupId: string,
+  alertDelay: number,
   newAlerts: Record<string, Alert<State, Context, ActionGroupIds>> = {},
   activeAlerts: Record<string, Alert<State, Context, ActionGroupIds>> = {},
   recoveredAlerts: Record<string, Alert<State, Context, RecoveryActionGroupId>> = {},
-  currentRecoveredAlerts: Record<string, Alert<State, Context, RecoveryActionGroupId>> = {}
+  currentRecoveredAlerts: Record<string, Alert<State, Context, RecoveryActionGroupId>> = {},
+  startedAt?: string | null
 ) {
   const currentActiveAlerts: Record<string, Alert<State, Context, ActionGroupIds>> = {};
 
   for (const id of keys(activeAlerts)) {
     const alert = activeAlerts[id];
     alert.incrementActiveCount();
     alert.resetPendingRecoveredCount();
-    currentActiveAlerts[id] = alert;
+    // do not trigger an action notification if the number of consecutive
+    // active alerts is less than the rule alertDelay threshold
+    if (alert.getActiveCount() < alertDelay) {
+      // remove from new alerts
+      delete newAlerts[id];
+    } else {
+      currentActiveAlerts[id] = alert;
+      // if the active count is equal to the alertDelay it is considered a new alert
+      if (alert.getActiveCount() === alertDelay) {
+        const currentTime = startedAt ?? new Date().toISOString();
+        const state = alert.getState();
+        // keep the state and update the start time and duration
+        alert.replaceState({ ...state, start: currentTime, duration: '0' });
+        newAlerts[id] = alert;
+      }
+    }
   }
 
   for (const id of keys(currentRecoveredAlerts)) {