From 5c0c8bfa2c93959b4c18a74ea29dba6501869132 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 6 Nov 2020 20:11:34 +0100 Subject: [PATCH] [Lens] Implement time scaling function (#82104) (#82854) --- src/plugins/data/common/search/aggs/types.ts | 2 +- .../aggs/utils/time_column_meta.test.ts | 41 +- .../search/aggs/utils/time_column_meta.ts | 10 +- .../data/public/search/expressions/esaggs.ts | 7 +- .../public/indexpattern_datasource/index.ts | 22 +- .../indexpattern_datasource/indexpattern.tsx | 1 + .../time_scale.test.ts | 368 ++++++++++++++++++ .../indexpattern_datasource/time_scale.ts | 167 ++++++++ 8 files changed, 601 insertions(+), 17 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/time_scale.test.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index 09a13762d4d704..897b60e91b100b 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -94,7 +94,7 @@ export interface AggsCommonStart { */ getDateMetaByDatatableColumn: ( column: DatatableColumn - ) => Promise; + ) => Promise; createAggConfigs: ( indexPattern: IndexPattern, configStates?: CreateAggConfigParams[], diff --git a/src/plugins/data/common/search/aggs/utils/time_column_meta.test.ts b/src/plugins/data/common/search/aggs/utils/time_column_meta.test.ts index e56d6227345547..8eb076f5b79065 100644 --- a/src/plugins/data/common/search/aggs/utils/time_column_meta.test.ts +++ b/src/plugins/data/common/search/aggs/utils/time_column_meta.test.ts @@ -91,6 +91,43 @@ describe('getDateMetaByDatatableColumn', () => { }); }); + it('throws if unable to resolve interval', async () => { + await expect( + getDateMetaByDatatableColumn(params)({ + id: 'test', + name: 'test', + meta: { + type: 'date', + source: 'esaggs', + sourceParams: { + type: BUCKET_TYPES.DATE_HISTOGRAM, + params: { + time_zone: 'UTC', + interval: 'auto', + }, + }, + }, + }) + ).rejects.toBeDefined(); + + await expect( + getDateMetaByDatatableColumn(params)({ + id: 'test', + name: 'test', + meta: { + type: 'date', + source: 'esaggs', + sourceParams: { + type: BUCKET_TYPES.DATE_HISTOGRAM, + params: { + time_zone: 'UTC', + }, + }, + }, + }) + ).rejects.toBeDefined(); + }); + it('returns resolved auto interval', async () => { expect( await getDateMetaByDatatableColumn(params)({ @@ -106,8 +143,8 @@ describe('getDateMetaByDatatableColumn', () => { interval: 'auto', }, appliedTimeRange: { - from: 'now-5d', - to: 'now', + from: '2020-10-05T00:00:00.000Z', + to: '2020-10-10T00:00:00.000Z', }, }, }, diff --git a/src/plugins/data/common/search/aggs/utils/time_column_meta.ts b/src/plugins/data/common/search/aggs/utils/time_column_meta.ts index 1bea716c6a0490..7ed8cb79f63f47 100644 --- a/src/plugins/data/common/search/aggs/utils/time_column_meta.ts +++ b/src/plugins/data/common/search/aggs/utils/time_column_meta.ts @@ -38,11 +38,11 @@ export const getDateMetaByDatatableColumn = ({ getConfig, }: DateMetaByColumnDeps) => async ( column: DatatableColumn -): Promise => { +): Promise => { if (column.meta.source !== 'esaggs') return; if (column.meta.sourceParams?.type !== BUCKET_TYPES.DATE_HISTOGRAM) return; const params = column.meta.sourceParams.params as AggParamsDateHistogram; - const appliedTimeRange = column.meta.sourceParams.appliedTimeRange as TimeRange; + const appliedTimeRange = column.meta.sourceParams.appliedTimeRange as TimeRange | undefined; const tz = inferTimeZone( params, @@ -52,9 +52,11 @@ export const getDateMetaByDatatableColumn = ({ ); const interval = - params.interval === 'auto' ? calculateAutoTimeExpression(appliedTimeRange) : params.interval; + params.interval === 'auto' && appliedTimeRange + ? calculateAutoTimeExpression(appliedTimeRange) + : params.interval; - if (!interval) { + if (!interval || interval === 'auto') { throw new Error('time interval could not be determined'); } diff --git a/src/plugins/data/public/search/expressions/esaggs.ts b/src/plugins/data/public/search/expressions/esaggs.ts index dba77d398c8b67..3932484801fa81 100644 --- a/src/plugins/data/public/search/expressions/esaggs.ts +++ b/src/plugins/data/public/search/expressions/esaggs.ts @@ -267,6 +267,8 @@ export const esaggs = (): EsaggsExpressionFunctionDefinition => ({ searchSource.setField('index', indexPattern); searchSource.setField('size', 0); + const resolvedTimeRange = input?.timeRange && calculateBounds(input.timeRange); + const response = await handleCourierRequest({ searchSource, aggs, @@ -303,7 +305,10 @@ export const esaggs = (): EsaggsExpressionFunctionDefinition => ({ input?.timeRange && args.timeFields && args.timeFields.includes(column.aggConfig.params.field?.name) - ? { from: input.timeRange.from, to: input.timeRange.to } + ? { + from: resolvedTimeRange?.min?.toISOString(), + to: resolvedTimeRange?.max?.toISOString(), + } : undefined, ...column.aggConfig.serialize(), }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index 35987656f66703..92280b0fb6ce6f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -33,19 +33,23 @@ export class IndexPatternDatasource { { expressions, editorFrame, charts }: IndexPatternDatasourceSetupPlugins ) { editorFrame.registerDatasource(async () => { - const { getIndexPatternDatasource, renameColumns, formatColumn } = await import( - '../async_services' - ); - expressions.registerFunction(renameColumns); - expressions.registerFunction(formatColumn); - return core.getStartServices().then(([coreStart, { data }]) => - getIndexPatternDatasource({ + const { + getIndexPatternDatasource, + renameColumns, + formatColumn, + getTimeScaleFunction, + } = await import('../async_services'); + return core.getStartServices().then(([coreStart, { data }]) => { + expressions.registerFunction(getTimeScaleFunction(data)); + expressions.registerFunction(renameColumns); + expressions.registerFunction(formatColumn); + return getIndexPatternDatasource({ core: coreStart, storage: new Storage(localStorage), data, charts, - }) - ) as Promise; + }); + }) as Promise; }); } } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 0d822927808084..ecca1b878e9a7d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -76,6 +76,7 @@ export function columnToOperation(column: IndexPatternColumn, uniqueLabel?: stri export * from './rename_columns'; export * from './format_column'; +export * from './time_scale'; export function getIndexPatternDatasource({ core, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.test.ts new file mode 100644 index 00000000000000..c29e2cd9567dc8 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.test.ts @@ -0,0 +1,368 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import moment from 'moment'; +import { Datatable } from 'src/plugins/expressions/public'; +import { DataPublicPluginStart } from 'src/plugins/data/public'; +import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; +import { functionWrapper } from 'src/plugins/expressions/common/expression_functions/specs/tests/utils'; +import { getTimeScaleFunction, TimeScaleArgs } from './time_scale'; + +describe('time_scale', () => { + let timeScale: (input: Datatable, args: TimeScaleArgs) => Promise; + let dataMock: jest.Mocked; + + const emptyTable: Datatable = { + type: 'datatable', + columns: [ + { + id: 'date', + name: 'date', + meta: { + type: 'date', + }, + }, + { + id: 'metric', + name: 'metric', + meta: { + type: 'number', + }, + }, + ], + rows: [], + }; + + const defaultArgs: TimeScaleArgs = { + dateColumnId: 'date', + inputColumnId: 'metric', + outputColumnId: 'scaledMetric', + targetUnit: 'h', + }; + + beforeEach(() => { + dataMock = dataPluginMock.createStartContract(); + (dataMock.search.aggs.getDateMetaByDatatableColumn as jest.Mock).mockReturnValue({ + timeZone: 'UTC', + timeRange: { + from: '2020-10-05T00:00:00.000Z', + to: '2020-10-10T00:00:00.000Z', + }, + interval: '1d', + }); + (dataMock.query.timefilter.timefilter.calculateBounds as jest.Mock).mockImplementation( + ({ from, to }) => ({ + min: moment(from), + max: moment(to), + }) + ); + timeScale = functionWrapper(getTimeScaleFunction(dataMock)); + }); + + it('should apply time scale factor to each row', async () => { + const result = await timeScale( + { + ...emptyTable, + rows: [ + { + date: moment('2020-10-05T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-06T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-07T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-08T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-09T00:00:00.000Z').valueOf(), + metric: 24, + }, + ], + }, + { + ...defaultArgs, + } + ); + + expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([1, 1, 1, 1, 1]); + }); + + it('should skip gaps in the data', async () => { + const result = await timeScale( + { + ...emptyTable, + rows: [ + { + date: moment('2020-10-05T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-06T00:00:00.000Z').valueOf(), + }, + { + date: moment('2020-10-07T00:00:00.000Z').valueOf(), + }, + { + date: moment('2020-10-08T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-09T00:00:00.000Z').valueOf(), + metric: 24, + }, + ], + }, + { + ...defaultArgs, + } + ); + + expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([ + 1, + undefined, + undefined, + 1, + 1, + ]); + }); + + it('should return input unchanged if input column does not exist', async () => { + const mismatchedTable = { + ...emptyTable, + rows: [ + { + date: moment('2020-10-05T00:00:00.000Z').valueOf(), + metric: 24, + }, + ], + }; + const result = await timeScale(mismatchedTable, { + ...defaultArgs, + inputColumnId: 'nonexistent', + }); + + expect(result).toBe(mismatchedTable); + }); + + it('should be able to scale up as well', async () => { + (dataMock.search.aggs.getDateMetaByDatatableColumn as jest.Mock).mockReturnValue({ + timeZone: 'UTC', + timeRange: { + from: '2020-10-05T12:00:00.000Z', + to: '2020-10-05T16:00:00.000Z', + }, + interval: '1h', + }); + const result = await timeScale( + { + ...emptyTable, + rows: [ + { + date: moment('2020-10-05T12:00:00.000Z').valueOf(), + metric: 1, + }, + { + date: moment('2020-10-05T13:00:00.000Z').valueOf(), + metric: 1, + }, + { + date: moment('2020-10-05T14:00:00.000Z').valueOf(), + metric: 1, + }, + { + date: moment('2020-10-05T15:00:00.000Z').valueOf(), + metric: 1, + }, + ], + }, + { + ...defaultArgs, + targetUnit: 'd', + } + ); + + expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([24, 24, 24, 24]); + }); + + it('can scale starting from unit multiple target intervals', async () => { + (dataMock.search.aggs.getDateMetaByDatatableColumn as jest.Mock).mockReturnValue({ + timeZone: 'UTC', + timeRange: { + from: '2020-10-05T13:00:00.000Z', + to: '2020-10-05T23:00:00.000Z', + }, + interval: '3h', + }); + const result = await timeScale( + { + ...emptyTable, + rows: [ + { + // bucket is cut off by one hour because of the time range + date: moment('2020-10-05T12:00:00.000Z').valueOf(), + metric: 2, + }, + { + date: moment('2020-10-05T15:00:00.000Z').valueOf(), + metric: 3, + }, + { + date: moment('2020-10-05T18:00:00.000Z').valueOf(), + metric: 3, + }, + { + // bucket is cut off by one hour because of the time range + date: moment('2020-10-05T21:00:00.000Z').valueOf(), + metric: 2, + }, + ], + }, + { + ...defaultArgs, + targetUnit: 'h', + } + ); + + expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([1, 1, 1, 1]); + }); + + it('take start and end of timerange into account', async () => { + (dataMock.search.aggs.getDateMetaByDatatableColumn as jest.Mock).mockReturnValue({ + timeZone: 'UTC', + timeRange: { + from: '2020-10-05T12:00:00.000Z', + to: '2020-10-09T12:00:00.000Z', + }, + interval: '1d', + }); + const result = await timeScale( + { + ...emptyTable, + rows: [ + { + // this is a partial bucket because it starts before the start of the time range + date: moment('2020-10-05T00:00:00.000Z').valueOf(), + metric: 12, + }, + { + date: moment('2020-10-06T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-07T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-08T00:00:00.000Z').valueOf(), + metric: 24, + }, + { + // this is a partial bucket because it ends earlier than the regular interval of 1d + date: moment('2020-10-09T00:00:00.000Z').valueOf(), + metric: 12, + }, + ], + }, + { + ...defaultArgs, + } + ); + + expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([1, 1, 1, 1, 1]); + }); + + it('should respect DST switches', async () => { + (dataMock.search.aggs.getDateMetaByDatatableColumn as jest.Mock).mockReturnValue({ + timeZone: 'Europe/Berlin', + timeRange: { + from: '2020-10-23T00:00:00.000+02:00', + to: '2020-10-27T00:00:00.000+01:00', + }, + interval: '1d', + }); + const result = await timeScale( + { + ...emptyTable, + rows: [ + { + date: moment('2020-10-23T00:00:00.000+02:00').valueOf(), + metric: 24, + }, + { + date: moment('2020-10-24T00:00:00.000+02:00').valueOf(), + metric: 24, + }, + { + // this day has one hour more in Europe/Berlin due to DST switch + date: moment('2020-10-25T00:00:00.000+02:00').valueOf(), + metric: 25, + }, + { + date: moment('2020-10-26T00:00:00.000+01:00').valueOf(), + metric: 24, + }, + ], + }, + { + ...defaultArgs, + } + ); + + expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([1, 1, 1, 1]); + }); + + it('take leap years into account', async () => { + (dataMock.search.aggs.getDateMetaByDatatableColumn as jest.Mock).mockReturnValue({ + timeZone: 'UTC', + timeRange: { + from: '2010-01-01T00:00:00.000Z', + to: '2015-01-01T00:00:00.000Z', + }, + interval: '1y', + }); + const result = await timeScale( + { + ...emptyTable, + rows: [ + { + date: moment('2010-01-01T00:00:00.000Z').valueOf(), + metric: 365, + }, + { + date: moment('2011-01-01T00:00:00.000Z').valueOf(), + metric: 365, + }, + { + // 2012 is a leap year and has an additional day + date: moment('2012-01-01T00:00:00.000Z').valueOf(), + metric: 366, + }, + { + date: moment('2013-01-01T00:00:00.000Z').valueOf(), + metric: 365, + }, + { + date: moment('2014-01-01T00:00:00.000Z').valueOf(), + metric: 365, + }, + ], + }, + { + ...defaultArgs, + targetUnit: 'd', + } + ); + + expect(result.rows.map(({ scaledMetric }) => scaledMetric)).toEqual([1, 1, 1, 1, 1]); + }); +}); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts b/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts new file mode 100644 index 00000000000000..0937f40eeb6d3d --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/time_scale.ts @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import moment from 'moment-timezone'; +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition, Datatable } from 'src/plugins/expressions/public'; +import { DataPublicPluginStart } from 'src/plugins/data/public'; +import { search } from '../../../../../src/plugins/data/public'; + +type TimeScaleUnit = 's' | 'm' | 'h' | 'd'; + +export interface TimeScaleArgs { + dateColumnId: string; + inputColumnId: string; + outputColumnId: string; + targetUnit: TimeScaleUnit; + outputColumnName?: string; +} + +const unitInMs: Record = { + s: 1000, + m: 1000 * 60, + h: 1000 * 60 * 60, + d: 1000 * 60 * 60 * 24, +}; + +export function getTimeScaleFunction(data: DataPublicPluginStart) { + const timeScale: ExpressionFunctionDefinition< + 'lens_time_scale', + Datatable, + TimeScaleArgs, + Promise + > = { + name: 'lens_time_scale', + type: 'datatable', + help: '', + args: { + dateColumnId: { + types: ['string'], + help: '', + required: true, + }, + inputColumnId: { + types: ['string'], + help: '', + required: true, + }, + outputColumnId: { + types: ['string'], + help: '', + required: true, + }, + outputColumnName: { + types: ['string'], + help: '', + }, + targetUnit: { + types: ['string'], + options: ['s', 'm', 'h', 'd'], + help: '', + required: true, + }, + }, + inputTypes: ['datatable'], + async fn( + input, + { dateColumnId, inputColumnId, outputColumnId, outputColumnName, targetUnit }: TimeScaleArgs + ) { + if (input.columns.some((column) => column.id === outputColumnId)) { + throw new Error( + i18n.translate('xpack.lens.functions.timeScale.columnConflictMessage', { + defaultMessage: 'Specified outputColumnId {columnId} already exists.', + values: { + columnId: outputColumnId, + }, + }) + ); + } + + const dateColumnDefinition = input.columns.find((column) => column.id === dateColumnId); + + if (!dateColumnDefinition) { + throw new Error( + i18n.translate('xpack.lens.functions.timeScale.dateColumnMissingMessage', { + defaultMessage: 'Specified dateColumnId {columnId} does not exist.', + values: { + columnId: dateColumnId, + }, + }) + ); + } + + const inputColumnDefinition = input.columns.find((column) => column.id === inputColumnId); + + if (!inputColumnDefinition) { + return input; + } + + const outputColumnDefinition = { + ...inputColumnDefinition, + id: outputColumnId, + name: outputColumnName || outputColumnId, + }; + + const resultColumns = [...input.columns]; + // add output column after input column in the table + resultColumns.splice( + resultColumns.indexOf(inputColumnDefinition) + 1, + 0, + outputColumnDefinition + ); + + const targetUnitInMs = unitInMs[targetUnit]; + const timeInfo = await data.search.aggs.getDateMetaByDatatableColumn(dateColumnDefinition); + const intervalDuration = timeInfo && search.aggs.parseInterval(timeInfo.interval); + + if (!timeInfo || !intervalDuration) { + throw new Error( + i18n.translate('xpack.lens.functions.timeScale.timeInfoMissingMessage', { + defaultMessage: 'Could not fetch date histogram information', + }) + ); + } + // the datemath plugin always parses dates by using the current default moment time zone. + // to use the configured time zone, we are switching just for the bounds calculation. + const defaultTimezone = moment().zoneName(); + moment.tz.setDefault(timeInfo.timeZone); + + const timeBounds = + timeInfo.timeRange && data.query.timefilter.timefilter.calculateBounds(timeInfo.timeRange); + + const result = { + ...input, + columns: resultColumns, + rows: input.rows.map((row) => { + const newRow = { ...row }; + + let startOfBucket = moment(row[dateColumnId]); + let endOfBucket = startOfBucket.clone().add(intervalDuration); + if (timeBounds && timeBounds.min) { + startOfBucket = moment.max(startOfBucket, timeBounds.min); + } + if (timeBounds && timeBounds.max) { + endOfBucket = moment.min(endOfBucket, timeBounds.max); + } + const bucketSize = endOfBucket.diff(startOfBucket); + const factor = bucketSize / targetUnitInMs; + + const currentValue = newRow[inputColumnId]; + if (currentValue != null) { + newRow[outputColumnId] = Number(currentValue) / factor; + } + + return newRow; + }), + }; + // reset default moment timezone + moment.tz.setDefault(defaultTimezone); + + return result; + }, + }; + return timeScale; +}