[ResponseOps][BE] Alert creation delay based on user definition (elas…

…tic#174657)

Related to elastic#173009

## Summary

This is the first of two PRs and only focuses on the backend
implementation. This PR adds a new `notificationDelay` field to the
`Rule` object. With the delay the rule will run X times and has to match
the threshold X times before triggering actions. It won't affect the
alert recovery, but it can be expanded on easily if we want to include
recovered alerts in the future.


### Checklist

- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios


### To verify

- Use [Dev Tools](http://localhost:5601/app/dev_tools#/console) to
create a rule with the `notificationDelay`

```
POST kbn:/api/alerting/rule
{
  "params": {
    "searchType": "esQuery",
    "timeWindowSize": 5,
    "timeWindowUnit": "m",
    "threshold": [
      -1
    ],
    "thresholdComparator": ">",
    "size": 100,
    "esQuery": """{
    "query":{
      "match_all" : {}
    }
  }""",
    "aggType": "count",
    "groupBy": "all",
    "termSize": 5,
    "excludeHitsFromPreviousRun": false,
    "sourceFields": [],
    "index": [
      ".kibana-event-log*"
    ],
    "timeField": "@timestamp"
  },
  "consumer": "stackAlerts",
  "schedule": {
    "interval": "1m"
  },
  "tags": [],
  "name": "test",
  "rule_type_id": ".es-query",
  "actions": [
    {
      "group": "query matched",
      "id": "${ACTION_ID}",
      "params": {
        "level": "info",
        "message": """Elasticsearch query rule '{{rule.name}}' is active:

- Value: {{context.value}}
- Conditions Met: {{context.conditions}} over {{rule.params.timeWindowSize}}{{rule.params.timeWindowUnit}}
- Timestamp: {{context.date}}
- Link: {{context.link}}"""
      },
      "frequency": {
        "notify_when": "onActionGroupChange",
        "throttle": null,
        "summary": false
      }
    }
  ],
  "notification_delay": {
    "active": 3
  }
}
```

- Verify that the rule will not trigger actions until it has matched the
delay threshold. It might be helpful to look at rule details page and
add the Triggered actions column to easily see the action was triggered
after X consecutive active alerts
<img width="1420" alt="Screen Shot 2024-01-16 at 1 18 52 PM"
src="https://github.com/elastic/kibana/assets/109488926/85d8ceef-042c-4a52-950e-24492dc0e79f">
- Verify that the delay does not affect recovered alerts

x-pack/packages/kbn-alerting-state-types/src/lifecycle_state.ts

@@ -17,6 +17,8 @@ const trackedAlertStateRt = t.type({
   flappingHistory: t.array(t.boolean),
   // flapping flag that indicates whether the alert is flapping
   flapping: t.boolean,
+  // count of consecutive recovered alerts for flapping
+  // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
   pendingRecoveredCount: t.number,
 });
 

x-pack/packages/kbn-alerting-state-types/src/task_state/v1/schema.ts

@@ -32,8 +32,13 @@ export const metaSchema = schema.object({
   // flapping flag that indicates whether the alert is flapping
   flapping: schema.maybe(schema.boolean()),
   maintenanceWindowIds: schema.maybe(schema.arrayOf(schema.string())),
+  // count of consecutive recovered alerts for flapping
+  // will reset if the alert is active or if equal to the statusChangeThreshold stored in the rule settings
   pendingRecoveredCount: schema.maybe(schema.number()),
   uuid: schema.maybe(schema.string()),
+  // count of consecutive active alerts
+  // will reset if the alert is recovered or if equal to notificationDelay.active stored in the rule
+  activeCount: schema.maybe(schema.number()),
 });
 
 export const rawAlertInstanceSchema = schema.object({

x-pack/plugins/alerting/common/routes/rule/apis/create/schemas/v1.ts

@@ -7,7 +7,7 @@
 
 import { schema } from '@kbn/config-schema';
 import { validateDurationV1, validateHoursV1, validateTimezoneV1 } from '../../../validation';
-import { notifyWhenSchemaV1 } from '../../../response';
+import { notifyWhenSchemaV1, notificationDelaySchemaV1 } from '../../../response';
 import { alertsFilterQuerySchemaV1 } from '../../../../alerts_filter_query';
 
 export const actionFrequencySchema = schema.object({
@@ -68,6 +68,7 @@ export const createBodySchema = schema.object({
   }),
   actions: schema.arrayOf(actionSchema, { defaultValue: [] }),
   notify_when: schema.maybe(schema.nullable(notifyWhenSchemaV1)),
+  notification_delay: schema.maybe(notificationDelaySchemaV1),
 });
 
 export const createParamsSchema = schema.object({

x-pack/plugins/alerting/common/routes/rule/apis/create/types/v1.ts

@@ -30,6 +30,7 @@ export interface CreateRuleRequestBody<Params extends RuleParamsV1 = never> {
   schedule: CreateBodySchema['schedule'];
   actions: CreateBodySchema['actions'];
   notify_when?: CreateBodySchema['notify_when'];
+  notification_delay?: CreateBodySchema['notification_delay'];
 }
 
 export interface CreateRuleResponse<Params extends RuleParamsV1 = never> {

x-pack/plugins/alerting/common/routes/rule/response/index.ts

@@ -37,6 +37,7 @@ export {
   ruleSnoozeScheduleSchema as ruleSnoozeScheduleSchemaV1,
   notifyWhenSchema as notifyWhenSchemaV1,
   scheduleIdsSchema as scheduleIdsSchemaV1,
+  notificationDelaySchema as notificationDelaySchemaV1,
 } from './schemas/v1';
 
 export type {

x-pack/plugins/alerting/common/routes/rule/response/schemas/v1.ts

@@ -182,6 +182,10 @@ export const ruleSnoozeScheduleSchema = schema.object({
   skipRecurrences: schema.maybe(schema.arrayOf(schema.string())),
 });
 
+export const notificationDelaySchema = schema.object({
+  active: schema.number(),
+});
+
 export const ruleResponseSchema = schema.object({
   id: schema.string(),
   enabled: schema.boolean(),
@@ -214,6 +218,7 @@ export const ruleResponseSchema = schema.object({
   revision: schema.number(),
   running: schema.maybe(schema.nullable(schema.boolean())),
   view_in_app_relative_url: schema.maybe(schema.nullable(schema.string())),
+  notification_delay: schema.maybe(notificationDelaySchema),
 });
 
 export const scheduleIdsSchema = schema.maybe(schema.arrayOf(schema.string()));

x-pack/plugins/alerting/common/routes/rule/response/types/v1.ts

@@ -53,4 +53,5 @@ export interface RuleResponse<Params extends RuleParams = never> {
   revision: RuleResponseSchemaType['revision'];
   running?: RuleResponseSchemaType['running'];
   view_in_app_relative_url?: RuleResponseSchemaType['view_in_app_relative_url'];
+  notification_delay?: RuleResponseSchemaType['notification_delay'];
 }

x-pack/plugins/alerting/common/rule.ts

@@ -141,6 +141,10 @@ export interface MappedParamsProperties {
 
 export type MappedParams = SavedObjectAttributes & MappedParamsProperties;
 
+export interface NotificationDelay {
+  active: number;
+}
+
 export interface Rule<Params extends RuleTypeParams = never> {
   id: string;
   enabled: boolean;
@@ -174,6 +178,7 @@ export interface Rule<Params extends RuleTypeParams = never> {
   revision: number;
   running?: boolean | null;
   viewInAppRelativeUrl?: string;
+  notificationDelay?: NotificationDelay;
 }
 
 export interface SanitizedAlertsFilter extends AlertsFilter {

x-pack/plugins/alerting/server/alert/alert.test.ts

@@ -543,6 +543,7 @@ describe('toRaw', () => {
         },
         flappingHistory: [false, true, true],
         pendingRecoveredCount: 2,
+        activeCount: 1,
       },
     };
     const alertInstance = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>(
@@ -562,6 +563,7 @@ describe('toRaw', () => {
         },
         flappingHistory: [false, true, true],
         flapping: false,
+        activeCount: 1,
       },
     };
     const alertInstance = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>(
@@ -574,6 +576,7 @@ describe('toRaw', () => {
         flapping: false,
         maintenanceWindowIds: [],
         uuid: expect.any(String),
+        activeCount: 1,
       },
     });
   });
@@ -746,3 +749,43 @@ describe('isFilteredOut', () => {
     expect(alert.isFilteredOut(summarizedAlerts)).toBe(true);
   });
 });
+
+describe('incrementActiveCount', () => {
+  test('correctly increments activeCount', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
+      meta: { activeCount: 3 },
+    });
+    alert.incrementActiveCount();
+    expect(alert.getActiveCount()).toEqual(4);
+  });
+
+  test('correctly increments activeCount when it is not already defined', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1');
+    alert.incrementActiveCount();
+    expect(alert.getActiveCount()).toEqual(1);
+  });
+});
+
+describe('getActiveCount', () => {
+  test('returns ActiveCount', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
+      meta: { activeCount: 3 },
+    });
+    expect(alert.getActiveCount()).toEqual(3);
+  });
+
+  test('defines and returns activeCount when it is not already defined', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1');
+    expect(alert.getActiveCount()).toEqual(0);
+  });
+});
+
+describe('resetActiveCount', () => {
+  test('resets activeCount to 0', () => {
+    const alert = new Alert<AlertInstanceState, AlertInstanceContext, DefaultActionGroupId>('1', {
+      meta: { activeCount: 3 },
+    });
+    alert.resetActiveCount();
+    expect(alert.getActiveCount()).toEqual(0);
+  });
+});

x-pack/plugins/alerting/server/alert/alert.ts

@@ -255,6 +255,7 @@ export class Alert<
             flappingHistory: this.meta.flappingHistory,
             flapping: this.meta.flapping,
             uuid: this.meta.uuid,
+            activeCount: this.meta.activeCount,
           },
         }
       : {
@@ -327,4 +328,19 @@ export class Alert<
   getMaintenanceWindowIds() {
     return this.meta.maintenanceWindowIds ?? [];
   }
+
+  incrementActiveCount() {
+    if (!this.meta.activeCount) {
+      this.meta.activeCount = 0;
+    }
+    this.meta.activeCount++;
+  }
+
+  getActiveCount() {
+    return this.meta.activeCount || 0;
+  }
+
+  resetActiveCount() {
+    this.meta.activeCount = 0;
+  }
 }

x-pack/plugins/alerting/server/application/rule/methods/create/schemas/create_rule_data_schema.ts

@@ -7,7 +7,11 @@
 
 import { schema } from '@kbn/config-schema';
 import { validateDuration } from '../../../validation';
-import { notifyWhenSchema, actionAlertsFilterSchema } from '../../../schemas';
+import {
+  notifyWhenSchema,
+  actionAlertsFilterSchema,
+  notificationDelaySchema,
+} from '../../../schemas';
 
 export const createRuleDataSchema = schema.object({
   name: schema.string(),
@@ -40,4 +44,5 @@ export const createRuleDataSchema = schema.object({
     { defaultValue: [] }
   ),
   notifyWhen: schema.maybe(schema.nullable(notifyWhenSchema)),
+  notificationDelay: schema.maybe(notificationDelaySchema),
 });

x-pack/plugins/alerting/server/application/rule/methods/create/types/create_rule_data.ts

@@ -22,4 +22,5 @@ export interface CreateRuleData<Params extends RuleParams = never> {
   schedule: CreateRuleDataType['schedule'];
   actions: CreateRuleDataType['actions'];
   notifyWhen?: CreateRuleDataType['notifyWhen'];
+  notificationDelay?: CreateRuleDataType['notificationDelay'];
 }

x-pack/plugins/alerting/server/application/rule/schemas/index.ts

@@ -13,6 +13,7 @@ export {
   monitoringSchema,
   ruleSchema,
   ruleDomainSchema,
+  notificationDelaySchema,
 } from './rule_schemas';
 
 export {

x-pack/plugins/alerting/server/application/rule/schemas/rule_schemas.ts

@@ -132,6 +132,10 @@ export const snoozeScheduleSchema = schema.object({
   skipRecurrences: schema.maybe(schema.arrayOf(schema.string())),
 });
 
+export const notificationDelaySchema = schema.object({
+  active: schema.number(),
+});
+
 /**
  * Unsanitized (domain) rule schema, used by internal rules clients
  */
@@ -168,6 +172,7 @@ export const ruleDomainSchema = schema.object({
   revision: schema.number(),
   running: schema.maybe(schema.nullable(schema.boolean())),
   viewInAppRelativeUrl: schema.maybe(schema.nullable(schema.string())),
+  notificationDelay: schema.maybe(notificationDelaySchema),
 });
 
 /**
@@ -205,4 +210,5 @@ export const ruleSchema = schema.object({
   revision: schema.number(),
   running: schema.maybe(schema.nullable(schema.boolean())),
   viewInAppRelativeUrl: schema.maybe(schema.nullable(schema.string())),
+  notificationDelay: schema.maybe(notificationDelaySchema),
 });

x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_attributes_to_rule_domain.ts

@@ -216,6 +216,7 @@ export const transformRuleAttributesToRuleDomain = <Params extends RuleParams =
     ...(esRule.nextRun ? { nextRun: new Date(esRule.nextRun) } : {}),
     revision: esRule.revision,
     running: esRule.running,
+    ...(esRule.notificationDelay ? { notificationDelay: esRule.notificationDelay } : {}),
   };
 
   // Bad casts, but will fix once we fix all rule types

x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_domain_to_rule.ts

@@ -49,6 +49,7 @@ export const transformRuleDomainToRule = <Params extends RuleParams = never>(
     revision: ruleDomain.revision,
     running: ruleDomain.running,
     viewInAppRelativeUrl: ruleDomain.viewInAppRelativeUrl,
+    notificationDelay: ruleDomain.notificationDelay,
   };
 
   if (isPublic) {

x-pack/plugins/alerting/server/application/rule/transforms/transform_rule_domain_to_rule_attributes.ts

@@ -68,5 +68,6 @@ export const transformRuleDomainToRuleAttributes = (
     ...(rule.nextRun !== undefined ? { nextRun: rule.nextRun?.toISOString() || null } : {}),
     revision: rule.revision,
     ...(rule.running !== undefined ? { running: rule.running } : {}),
+    ...(rule.notificationDelay !== undefined ? { notificationDelay: rule.notificationDelay } : {}),
   };
 };

x-pack/plugins/alerting/server/application/rule/types/rule.ts

@@ -85,6 +85,7 @@ export interface Rule<Params extends RuleParams = never> {
   revision: RuleSchemaType['revision'];
   running?: RuleSchemaType['running'];
   viewInAppRelativeUrl?: RuleSchemaType['viewInAppRelativeUrl'];
+  notificationDelay?: RuleSchemaType['notificationDelay'];
 }
 
 export interface RuleDomain<Params extends RuleParams = never> {
@@ -120,4 +121,5 @@ export interface RuleDomain<Params extends RuleParams = never> {
   revision: RuleDomainSchemaType['revision'];
   running?: RuleDomainSchemaType['running'];
   viewInAppRelativeUrl?: RuleDomainSchemaType['viewInAppRelativeUrl'];
+  notificationDelay?: RuleSchemaType['notificationDelay'];
 }

x-pack/plugins/alerting/server/data/rule/types/rule_attributes.ts

@@ -142,6 +142,10 @@ interface RuleMetaAttributes {
   versionApiKeyLastmodified?: string;
 }
 
+interface NotificationDelayAttributes {
+  active: number;
+}
+
 export interface RuleAttributes {
   name: string;
   tags: string[];
@@ -174,4 +178,5 @@ export interface RuleAttributes {
   nextRun?: string | null;
   revision: number;
   running?: boolean | null;
+  notificationDelay?: NotificationDelayAttributes;
 }