diff --git a/x-pack/packages/kbn-data-forge/src/constants.ts b/x-pack/packages/kbn-data-forge/src/constants.ts index d96735eeff8d01..0b5ab1978d9832 100644 --- a/x-pack/packages/kbn-data-forge/src/constants.ts +++ b/x-pack/packages/kbn-data-forge/src/constants.ts @@ -34,4 +34,5 @@ export const DEFAULTS = { EVENT_TEMPLATE: 'good', REDUCE_WEEKEND_TRAFFIC_BY: 0, EPHEMERAL_PROJECT_IDS: 0, + ALIGN_EVENTS_TO_INTERVAL: 0, }; diff --git a/x-pack/packages/kbn-data-forge/src/lib/cli_to_partial_config.ts b/x-pack/packages/kbn-data-forge/src/lib/cli_to_partial_config.ts index 7a355a61840c19..b116f262303db3 100644 --- a/x-pack/packages/kbn-data-forge/src/lib/cli_to_partial_config.ts +++ b/x-pack/packages/kbn-data-forge/src/lib/cli_to_partial_config.ts @@ -47,6 +47,7 @@ export function cliOptionsToPartialConfig(options: CliOptions) { concurrency: options.concurrency, reduceWeekendTrafficBy: options.reduceWeekendTrafficBy, ephemeralProjectIds: options.ephemeralProjectIds, + alignEventsToInterval: options.alignEventsToInterval === true, }, schedule: [schedule], }; diff --git a/x-pack/packages/kbn-data-forge/src/lib/create_config.ts b/x-pack/packages/kbn-data-forge/src/lib/create_config.ts index d6d88e9acb9fff..c8a48b26329984 100644 --- a/x-pack/packages/kbn-data-forge/src/lib/create_config.ts +++ b/x-pack/packages/kbn-data-forge/src/lib/create_config.ts @@ -62,6 +62,7 @@ export function createConfig(partialConfig: PartialConfig = {}) { concurrency: DEFAULTS.CONCURRENCY, reduceWeekendTrafficBy: DEFAULTS.REDUCE_WEEKEND_TRAFFIC_BY, ephemeralProjectIds: DEFAULTS.EPHEMERAL_PROJECT_IDS, + alignEventsToInterval: DEFAULTS.ALIGN_EVENTS_TO_INTERVAL === 1, ...(partialConfig.indexing ?? {}), }, schedule: partialConfig.schedule ?? [schedule], diff --git a/x-pack/packages/kbn-data-forge/src/lib/create_events.ts b/x-pack/packages/kbn-data-forge/src/lib/create_events.ts index 7827b996bfd60d..5c10df31539836 100644 --- a/x-pack/packages/kbn-data-forge/src/lib/create_events.ts +++ b/x-pack/packages/kbn-data-forge/src/lib/create_events.ts @@ -83,22 +83,31 @@ export async function createEvents( ); epc = epc * (1 - config.indexing.reduceWeekendTrafficBy); } - // range(epc).map((i) => { - // const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs; - // const eventTimestamp = moment(random(currentTimestamp.valueOf(), currentTimestamp.valueOf() + interval)); - // return generateEvent(config, schedule, i, eventTimestamp); - // }).flat().forEach((event) => queue.push(event)); - range(epc) - .map(() => - moment(random(currentTimestamp.valueOf(), currentTimestamp.valueOf() + interval - 1)) - ) - .sort() - .map((ts, i) => { - const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs; - return generateEvent(config, schedule, i, ts); - }) - .flat() - .forEach((event) => queue.push(event)); + + // When --align-events-to-interval is set, we will index all the events on the same + // timestamp. Otherwise they will be distributed across the interval randomly. + if (config.indexing.alignEventsToInterval) { + range(epc) + .map((i) => { + const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs; + return generateEvent(config, schedule, i, currentTimestamp); + }) + .flat() + .forEach((event) => queue.push(event)); + } else { + range(epc) + .map(() => + moment(random(currentTimestamp.valueOf(), currentTimestamp.valueOf() + interval - 1)) + ) + .sort() + .map((ts, i) => { + const generateEvent = generateEvents[config.indexing.dataset] || generateEvents.fake_logs; + return generateEvent(config, schedule, i, ts); + }) + .flat() + .forEach((event) => queue.push(event)); + } + await queue.drain(); } else { logger.info({ took: 0, latency: 0, indexed: 0 }, 'Indexing 0 documents.'); diff --git a/x-pack/packages/kbn-data-forge/src/lib/parse_cli_options.ts b/x-pack/packages/kbn-data-forge/src/lib/parse_cli_options.ts index 8120b1bfb5be94..5d7fb4287236d9 100644 --- a/x-pack/packages/kbn-data-forge/src/lib/parse_cli_options.ts +++ b/x-pack/packages/kbn-data-forge/src/lib/parse_cli_options.ts @@ -74,6 +74,10 @@ export function parseCliOptions(): CliOptions { '--install-kibana-assets', 'This will install index patterns, visualizations, and dashboards for the dataset' ) + .option( + '--align-events-to-interval', + 'This will index all the events on the interval instead of randomly distributing them.' + ) .option( '--event-template