[Obs-UX-Mgmt] Fixing flaky test for Custom Threshold rule (#175479)

## Summary This PR fixes #175407 by increasing the rule lookback to 5 minutes to try and avoid picking up 2 buckets since we can't control the exact time of the rule execution to ensure accuracy and consistency. 😦 Fixes #175360 [Flaky Test Runner Results](https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4959) --------- Co-authored-by: Maryam Saeidi <maryam.saeidi@elastic.co>
elastic · Jan 26, 2024 · c8ada33 · c8ada33
1 parent 5257607
commit c8ada33
Show file tree

Hide file tree

Showing 15 changed files with 120 additions and 53 deletions.
diff --git a/x-pack/packages/kbn-data-forge/src/constants.ts b/x-pack/packages/kbn-data-forge/src/constants.ts
@@ -34,4 +34,5 @@ export const DEFAULTS = {
   EVENT_TEMPLATE: 'good',
   REDUCE_WEEKEND_TRAFFIC_BY: 0,
   EPHEMERAL_PROJECT_IDS: 0,
+  ALIGN_EVENTS_TO_INTERVAL: 0,
 };
diff --git a/x-pack/packages/kbn-data-forge/src/lib/cli_to_partial_config.ts b/x-pack/packages/kbn-data-forge/src/lib/cli_to_partial_config.ts
@@ -47,6 +47,7 @@ export function cliOptionsToPartialConfig(options: CliOptions) {
       concurrency: options.concurrency,
       reduceWeekendTrafficBy: options.reduceWeekendTrafficBy,
       ephemeralProjectIds: options.ephemeralProjectIds,
+      alignEventsToInterval: options.alignEventsToInterval === true,
     },
     schedule: [schedule],
   };

diff --git a/x-pack/packages/kbn-data-forge/src/lib/create_config.ts b/x-pack/packages/kbn-data-forge/src/lib/create_config.ts
@@ -62,6 +62,7 @@ export function createConfig(partialConfig: PartialConfig = {}) {
       concurrency: DEFAULTS.CONCURRENCY,
       reduceWeekendTrafficBy: DEFAULTS.REDUCE_WEEKEND_TRAFFIC_BY,
       ephemeralProjectIds: DEFAULTS.EPHEMERAL_PROJECT_IDS,
+      alignEventsToInterval: DEFAULTS.ALIGN_EVENTS_TO_INTERVAL === 1,
       ...(partialConfig.indexing ?? {}),
     },
     schedule: partialConfig.schedule ?? [schedule],

diff --git a/x-pack/packages/kbn-data-forge/src/lib/create_events.ts b/x-pack/packages/kbn-data-forge/src/lib/create_events.ts
@@ -83,22 +83,31 @@ export async function createEvents(
       );
       epc = epc * (1 - config.indexing.reduceWeekendTrafficBy);
     }
-    // range(epc).map((i) => {
-    //   const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs;
-    //   const eventTimestamp = moment(random(currentTimestamp.valueOf(), currentTimestamp.valueOf() + interval));
-    //   return generateEvent(config, schedule, i, eventTimestamp);
-    // }).flat().forEach((event) => queue.push(event));
-    range(epc)
-      .map(() =>
-        moment(random(currentTimestamp.valueOf(), currentTimestamp.valueOf() + interval - 1))
-      )
-      .sort()
-      .map((ts, i) => {
-        const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs;
-        return generateEvent(config, schedule, i, ts);
-      })
-      .flat()
-      .forEach((event) => queue.push(event));
+
+    // When --align-events-to-interval is set, we will index all the events on the same
+    // timestamp. Otherwise they will be distributed across the interval randomly.
+    if (config.indexing.alignEventsToInterval) {
+      range(epc)
+        .map((i) => {
+          const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs;
+          return generateEvent(config, schedule, i, currentTimestamp);
+        })
+        .flat()
+        .forEach((event) => queue.push(event));
+    } else {
+      range(epc)
+        .map(() =>
+          moment(random(currentTimestamp.valueOf(), currentTimestamp.valueOf() + interval - 1))
+        )
+        .sort()
+        .map((ts, i) => {
+          const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs;
+          return generateEvent(config, schedule, i, ts);
+        })
+        .flat()
+        .forEach((event) => queue.push(event));
+    }
+
     await queue.drain();
   } else {
     logger.info({ took: 0, latency: 0, indexed: 0 }, 'Indexing 0 documents.');

diff --git a/x-pack/packages/kbn-data-forge/src/lib/parse_cli_options.ts b/x-pack/packages/kbn-data-forge/src/lib/parse_cli_options.ts
@@ -74,6 +74,10 @@ export function parseCliOptions(): CliOptions {
       '--install-kibana-assets',
       'This will install index patterns, visualizations, and dashboards for the dataset'
     )
+    .option(
+      '--align-events-to-interval',
+      'This will index all the events on the interval instead of randomly distributing them.'
+    )
     .option(
       '--event-template <template>',
       'The name of the event template',

diff --git a/x-pack/packages/kbn-data-forge/src/types/index.ts b/x-pack/packages/kbn-data-forge/src/types/index.ts
@@ -119,6 +119,7 @@ export const ConfigRT = rt.type({
     concurrency: rt.number,
     reduceWeekendTrafficBy: rt.number,
     ephemeralProjectIds: rt.number,
+    alignEventsToInterval: rt.boolean,
   }),
   schedule: rt.array(ScheduleRT),
 });
@@ -177,4 +178,5 @@ export interface CliOptions {
   eventTemplate: string;
   reduceWeekendTrafficBy: number;
   ephemeralProjectIds: number;
+  alignEventsToInterval: boolean;
 }
diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_pct_fired.ts
@@ -36,18 +36,18 @@ export default function ({ getService }: FtrProviderContext) {
   describe('Custom Threshold rule - AVG - PCT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
-    const DATE_VIEW_TITLE = 'kbn-data-forge-fake_hosts.fake_hosts-*';
-    const DATE_VIEW_NAME = 'ad-hoc-data-view-name';
+    const DATA_VIEW_TITLE = 'kbn-data-forge-fake_hosts.fake_hosts-*';
+    const DATA_VIEW_NAME = 'ad-hoc-data-view-name';
     const DATA_VIEW_ID = 'data-view-id';
     const MOCKED_AD_HOC_DATA_VIEW = {
       id: DATA_VIEW_ID,
-      title: DATE_VIEW_TITLE,
+      title: DATA_VIEW_TITLE,
       timeFieldName: '@timestamp',
       sourceFilters: [],
       fieldFormats: {},
       runtimeFieldMap: {},
       allowNoIndex: false,
-      name: DATE_VIEW_NAME,
+      name: DATA_VIEW_NAME,
       allowHidden: false,
     };
     let dataForgeConfig: PartialConfig;
@@ -70,13 +70,18 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
         ],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 10000 },
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 10000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       logger.info(JSON.stringify(dataForgeIndices.join(',')));
       await waitForDocumentInIndex({
         esClient,
-        indexName: DATE_VIEW_TITLE,
+        indexName: DATA_VIEW_TITLE,
         docCountTarget: 270,
       });
     });
@@ -238,7 +243,7 @@ export default function ({ getService }: FtrProviderContext) {
           `https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
         );
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `Average system.cpu.user.pct is 250%, above the threshold of 50%. (duration: 5 mins, data view: ${DATE_VIEW_NAME})`
+          `Average system.cpu.user.pct is 250%, above the threshold of 50%. (duration: 5 mins, data view: ${DATA_VIEW_NAME})`
         );
         expect(resp.hits.hits[0]._source?.value).eql('250%');
 
@@ -248,7 +253,7 @@ export default function ({ getService }: FtrProviderContext) {
 
         expect(resp.hits.hits[0]._source?.viewInAppUrl).contain('LOG_EXPLORER_LOCATOR');
         expect(omit(parsedViewInAppUrl.params, 'timeRange.from')).eql({
-          dataset: DATE_VIEW_TITLE,
+          dataset: DATA_VIEW_TITLE,
           timeRange: { to: 'now' },
           query: { query: '', language: 'kuery' },
         });

diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/avg_us_fired.ts
@@ -40,7 +40,7 @@ export default function ({ getService }: FtrProviderContext) {
   describe('Custom Threshold rule - AVG - US - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
-    const DATE_VIEW = 'traces-apm*,metrics-apm*,logs-apm*';
+    const DATA_VIEW = 'traces-apm*,metrics-apm*,logs-apm*';
     const DATA_VIEW_ID = 'data-view-id';
     const DATA_VIEW_NAME = 'test-data-view-name';
 
@@ -57,7 +57,7 @@ export default function ({ getService }: FtrProviderContext) {
         supertest,
         name: DATA_VIEW_NAME,
         id: DATA_VIEW_ID,
-        title: DATE_VIEW,
+        title: DATA_VIEW,
       });
     });
 

diff --git a/...alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts b/...alerting_api_integration/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts
@@ -34,8 +34,8 @@ export default function ({ getService }: FtrProviderContext) {
   describe('Custom Threshold rule - CUSTOM_EQ - AVG - BYTES - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
-    // DATE_VIEW should match the index template:
-    const DATE_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
+    // DATA_VIEW should match the index template:
+    const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
     const DATA_VIEW_ID = 'data-view-id';
     let dataForgeConfig: PartialConfig;
     let dataForgeIndices: string[];
@@ -46,21 +46,35 @@ export default function ({ getService }: FtrProviderContext) {
 
     before(async () => {
       dataForgeConfig = {
-        schedule: [{ template: 'good', start: 'now-15m', end: 'now+5m' }],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 60000 },
+        schedule: [
+          {
+            template: 'good',
+            start: 'now-15m',
+            end: 'now+10m',
+            metrics: [
+              { name: 'system.network.in.bytes', method: 'linear', start: 5, end: 5 },
+              { name: 'system.network.out.bytes', method: 'linear', start: 5, end: 5 },
+            ],
+          },
+        ],
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 60000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       await waitForDocumentInIndex({
         esClient,
-        indexName: dataForgeIndices.join(','),
-        docCountTarget: 60,
+        indexName: DATA_VIEW,
+        docCountTarget: 75,
       });
-
       await createDataView({
         supertest,
-        name: DATE_VIEW,
+        name: DATA_VIEW,
         id: DATA_VIEW_ID,
-        title: DATE_VIEW,
+        title: DATA_VIEW,
       });
     });
 
@@ -227,7 +241,7 @@ export default function ({ getService }: FtrProviderContext) {
           `https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
         );
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `Custom equation is 1, above the threshold of 0.9. (duration: 1 min, data view: ${DATE_VIEW})`
+          `Custom equation is 1, above the threshold of 0.9. (duration: 1 min, data view: ${DATA_VIEW})`
         );
         expect(resp.hits.hits[0]._source?.value).eql('1');
       });

diff --git a/...est/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts b/...est/alerting_api_integration/observability/custom_threshold_rule/documents_count_fired.ts
@@ -37,9 +37,9 @@ export default function ({ getService }: FtrProviderContext) {
   describe('Custom  Threshold rule - DOCUMENTS_COUNT - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
-    const DATE_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
+    const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
     const DATA_VIEW_ID = 'data-view-id';
-    const DATE_VIEW_NAME = 'data-view-name';
+    const DATA_VIEW_NAME = 'data-view-name';
     let dataForgeConfig: PartialConfig;
     let dataForgeIndices: string[];
     let actionId: string;
@@ -61,7 +61,12 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
         ],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 60000 },
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 60000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       await waitForDocumentInIndex({
@@ -71,9 +76,9 @@ export default function ({ getService }: FtrProviderContext) {
       });
       await createDataView({
         supertest,
-        name: DATE_VIEW_NAME,
+        name: DATA_VIEW_NAME,
         id: DATA_VIEW_ID,
-        title: DATE_VIEW,
+        title: DATA_VIEW,
       });
     });
 
@@ -96,8 +101,7 @@ export default function ({ getService }: FtrProviderContext) {
       await cleanup({ client: esClient, config: dataForgeConfig, logger });
     });
 
-    // FLAKY: https://github.com/elastic/kibana/issues/175407
-    describe.skip('Rule creation', () => {
+    describe('Rule creation', () => {
       it('creates rule successfully', async () => {
         actionId = await createIndexConnector({
           supertest,
@@ -237,8 +241,9 @@ export default function ({ getService }: FtrProviderContext) {
         expect(resp.hits.hits[0]._source?.alertDetailsUrl).eql(
           `https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
         );
+
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `Document count is 3, not between the threshold of 1 and 2. (duration: 1 min, data view: ${DATE_VIEW_NAME})`
+          `Document count is 3, not between the threshold of 1 and 2. (duration: 1 min, data view: ${DATA_VIEW_NAME})`
         );
         expect(resp.hits.hits[0]._source?.value).eql('3');
 

diff --git a/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts b/x-pack/test/alerting_api_integration/observability/custom_threshold_rule/group_by_fired.ts
@@ -34,7 +34,7 @@ export default function ({ getService }: FtrProviderContext) {
   describe('Custom  Threshold rule - GROUP_BY - FIRED', () => {
     const CUSTOM_THRESHOLD_RULE_ALERT_INDEX = '.alerts-observability.threshold.alerts-default';
     const ALERT_ACTION_INDEX = 'alert-action-threshold';
-    const DATE_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
+    const DATA_VIEW = 'kbn-data-forge-fake_hosts.fake_hosts-*';
     const DATA_VIEW_ID = 'data-view-id';
     let dataForgeConfig: PartialConfig;
     let dataForgeIndices: string[];
@@ -57,7 +57,12 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
         ],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 10000 },
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 10000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       await waitForDocumentInIndex({
@@ -67,9 +72,9 @@ export default function ({ getService }: FtrProviderContext) {
       });
       await createDataView({
         supertest,
-        name: DATE_VIEW,
+        name: DATA_VIEW,
         id: DATA_VIEW_ID,
-        title: DATE_VIEW,
+        title: DATA_VIEW,
       });
     });
 
@@ -257,7 +262,7 @@ export default function ({ getService }: FtrProviderContext) {
           `https://localhost:5601/app/observability/alerts?_a=(kuery:%27kibana.alert.uuid:%20%22${alertId}%22%27%2CrangeFrom:%27${rangeFrom}%27%2CrangeTo:now%2Cstatus:all)`
         );
         expect(resp.hits.hits[0]._source?.reason).eql(
-          `Average system.cpu.total.norm.pct is 80%, above the threshold of 20%. (duration: 1 min, data view: ${DATE_VIEW}, group: host-0,container-0)`
+          `Average system.cpu.total.norm.pct is 80%, above the threshold of 20%. (duration: 1 min, data view: ${DATA_VIEW}, group: host-0,container-0)`
         );
         expect(resp.hits.hits[0]._source?.value).eql('80%');
         expect(resp.hits.hits[0]._source?.host).eql(

diff --git a/...rverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_fired.ts b/...rverless/api_integration/test_suites/observability/custom_threshold_rule/avg_pct_fired.ts
@@ -46,7 +46,12 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
         ],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 10000 },
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 10000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW, docCountTarget: 360 });

diff --git a/..._integration/test_suites/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts b/..._integration/test_suites/observability/custom_threshold_rule/custom_eq_avg_bytes_fired.ts
@@ -53,7 +53,12 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
         ],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 10000 },
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 10000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW, docCountTarget: 360 });

diff --git a/.../api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts b/.../api_integration/test_suites/observability/custom_threshold_rule/documents_count_fired.ts
@@ -47,7 +47,12 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
         ],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 60000 },
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 60000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW, docCountTarget: 60 });

diff --git a/...verless/api_integration/test_suites/observability/custom_threshold_rule/group_by_fired.ts b/...verless/api_integration/test_suites/observability/custom_threshold_rule/group_by_fired.ts
@@ -57,7 +57,12 @@ export default function ({ getService }: FtrProviderContext) {
             ],
           },
         ],
-        indexing: { dataset: 'fake_hosts' as Dataset, eventsPerCycle: 1, interval: 10000 },
+        indexing: {
+          dataset: 'fake_hosts' as Dataset,
+          eventsPerCycle: 1,
+          interval: 10000,
+          alignEventsToInterval: true,
+        },
       };
       dataForgeIndices = await generate({ client: esClient, config: dataForgeConfig, logger });
       await alertingApi.waitForDocumentInIndex({ indexName: DATA_VIEW, docCountTarget: 360 });