From 7f75ebd801f08170710ae8c73c44e7aa0c70090f Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 19:49:33 -0800 Subject: [PATCH 1/8] feat: support count rate agg funtions --- packages/api/src/clickhouse/index.ts | 39 ++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index c1dfb446f..fcc17a1a4 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -61,6 +61,9 @@ export enum AggFn { AvgRate = 'avg_rate', Count = 'count', CountDistinct = 'count_distinct', + CountPerSec = 'count_per_sec', + CountPerMin = 'count_per_min', + CountPerHour = 'count_per_hour', Max = 'max', MaxRate = 'max_rate', Min = 'min', @@ -1078,6 +1081,18 @@ const buildEventSeriesQuery = async ({ throw new Error('Rate is not supported in logs chart'); } + const isCountFn = + aggFn === AggFn.Count || + aggFn === AggFn.CountPerSec || + aggFn === AggFn.CountPerMin || + aggFn === AggFn.CountPerHour; + + if (field == null && !isCountFn) { + throw new Error( + 'Field is required for all aggregation functions except Count', + ); + } + const tableName = getLogStreamTableName(tableVersion, teamId); const whereClause = await buildSearchQueryWhereCondition({ endTime, @@ -1086,18 +1101,11 @@ const buildEventSeriesQuery = async ({ startTime, }); - if (field == null && aggFn !== AggFn.Count) { - throw new Error( - 'Field is required for all aggregation functions except Count', - ); - } - const selectField = field != null ? buildSearchColumnName(propertyTypeMappingsModel.get(field), field) : ''; - const isCountFn = aggFn === AggFn.Count; const groupByColumnNames = groupBy.map(g => { const columnName = buildSearchColumnName( propertyTypeMappingsModel.get(g), @@ -1130,8 +1138,23 @@ const buildEventSeriesQuery = async ({ const label = SqlString.escape(`${aggFn}(${field})`); const selectClause = [ - isCountFn + aggFn === AggFn.Count ? 'toFloat64(count()) as data' + : aggFn === AggFn.CountPerSec + ? SqlString.format( + `divide(count(), age('ss', toDateTime(?), toDateTime(?))) as data`, + [startTime / 1000, endTime / 1000], + ) + : aggFn === AggFn.CountPerMin + ? SqlString.format( + `divide(count(), age('mi', toDateTime(?), toDateTime(?))) as data`, + [startTime / 1000, endTime / 1000], + ) + : aggFn === AggFn.CountPerHour + ? SqlString.format( + `divide(count(), age('hh', toDateTime(?), toDateTime(?))) as data`, + [startTime / 1000, endTime / 1000], + ) : aggFn === AggFn.Sum ? `toFloat64(sum(${selectField})) as data` : aggFn === AggFn.Avg From f2020da8b21e373493228a7a4ac9782ec8a7f9c5 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 20:12:33 -0800 Subject: [PATCH 2/8] feat: add types --- packages/app/src/types.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/app/src/types.ts b/packages/app/src/types.ts index dbd46101e..f5084add8 100644 --- a/packages/app/src/types.ts +++ b/packages/app/src/types.ts @@ -165,6 +165,9 @@ export type AggFn = | 'avg' | 'count_distinct' | 'count' + | 'count_per_sec' + | 'count_per_min' + | 'count_per_hour' | 'max_rate' | 'max' | 'min_rate' From 33be2c66482db07eb3705aa67e445b820bc1ff9a Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 23:20:36 -0800 Subject: [PATCH 3/8] fix: rate window should base on logs timestamp --- .../clickhouse/__tests__/clickhouse.test.ts | 47 ++++++++++++++++++- packages/api/src/clickhouse/index.ts | 15 ++---- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts index 1b56f1aa3..8c04e2eaf 100644 --- a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts +++ b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts @@ -382,6 +382,51 @@ Array [ "ts_bucket": 1641341100, }, ] +`); + + const tableData = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'table', + table: 'logs', + aggFn: clickhouse.AggFn.CountPerMin, + where: `runId:${runId}`, + groupBy: ['testGroup'], + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: undefined, + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + }) + ).data.map(d => { + return _.pick(d, ['group', 'series_0.data', 'ts_bucket', 'rank']); + }); + + expect(tableData.length).toEqual(2); + expect(tableData).toMatchInlineSnapshot(` +Array [ + Object { + "group": Array [ + "group1", + ], + "rank": "1", + "series_0.data": 0.75, + "ts_bucket": "0", + }, + Object { + "group": Array [ + "group2", + ], + "rank": "1", + "series_0.data": 0.75, + "ts_bucket": "0", + }, +] `); }); @@ -846,8 +891,6 @@ Array [ }), ); - mockLogsPropertyTypeMappingsModel({}); - mockSpyMetricPropertyTypeMappingsModel({ runId: 'string', host: 'string', diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index fcc17a1a4..325a0de18 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -1141,20 +1141,11 @@ const buildEventSeriesQuery = async ({ aggFn === AggFn.Count ? 'toFloat64(count()) as data' : aggFn === AggFn.CountPerSec - ? SqlString.format( - `divide(count(), age('ss', toDateTime(?), toDateTime(?))) as data`, - [startTime / 1000, endTime / 1000], - ) + ? "divide(count(), age('ss', MIN(timestamp), MAX(timestamp))) as data" : aggFn === AggFn.CountPerMin - ? SqlString.format( - `divide(count(), age('mi', toDateTime(?), toDateTime(?))) as data`, - [startTime / 1000, endTime / 1000], - ) + ? "divide(count(), age('mi', MIN(timestamp), MAX(timestamp))) as data" : aggFn === AggFn.CountPerHour - ? SqlString.format( - `divide(count(), age('hh', toDateTime(?), toDateTime(?))) as data`, - [startTime / 1000, endTime / 1000], - ) + ? "divide(count(), age('hh', MIN(timestamp), MAX(timestamp))) as data" : aggFn === AggFn.Sum ? `toFloat64(sum(${selectField})) as data` : aggFn === AggFn.Avg From 1455e2e03422f591ff41ccc5493348213cf409f7 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 23:25:24 -0800 Subject: [PATCH 4/8] docs: add changeset --- .changeset/breezy-seahorses-swim.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/breezy-seahorses-swim.md diff --git a/.changeset/breezy-seahorses-swim.md b/.changeset/breezy-seahorses-swim.md new file mode 100644 index 000000000..1d3b70a93 --- /dev/null +++ b/.changeset/breezy-seahorses-swim.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +feat: support count per sec/min/hr aggregation functions From 27a264a8226e67ff5e7b38d1ddfd845a22b012f6 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 7 Jan 2024 15:54:14 -0800 Subject: [PATCH 5/8] feat: handle granularity --- packages/api/src/clickhouse/index.ts | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index 325a0de18..d1ef15025 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -1141,11 +1141,32 @@ const buildEventSeriesQuery = async ({ aggFn === AggFn.Count ? 'toFloat64(count()) as data' : aggFn === AggFn.CountPerSec - ? "divide(count(), age('ss', MIN(timestamp), MAX(timestamp))) as data" + ? granularity + ? SqlString.format('divide(count(), ?) as data', [ + ms(granularity) / ms('1 second'), + ]) + : SqlString.format( + "divide(count(), age('ss', toDateTime(?), toDateTime(?))) as data", + [startTime / 1000, endTime / 1000], + ) : aggFn === AggFn.CountPerMin - ? "divide(count(), age('mi', MIN(timestamp), MAX(timestamp))) as data" + ? granularity + ? SqlString.format('divide(count(), ?) as data', [ + ms(granularity) / ms('1 minute'), + ]) + : SqlString.format( + "divide(count(), age('mi', toDateTime(?), toDateTime(?))) as data", + [startTime / 1000, endTime / 1000], + ) : aggFn === AggFn.CountPerHour - ? "divide(count(), age('hh', MIN(timestamp), MAX(timestamp))) as data" + ? granularity + ? SqlString.format('divide(count(), ?) as data', [ + ms(granularity) / ms('1 hour'), + ]) + : SqlString.format( + "divide(count(), age('hh', toDateTime(?), toDateTime(?))) as data", + [startTime / 1000, endTime / 1000], + ) : aggFn === AggFn.Sum ? `toFloat64(sum(${selectField})) as data` : aggFn === AggFn.Avg From 2b02354c367e334ef579e2594408ef1069308749 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 7 Jan 2024 16:07:31 -0800 Subject: [PATCH 6/8] test: update snapshot --- packages/api/src/clickhouse/__tests__/clickhouse.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts index 8c04e2eaf..385526e5d 100644 --- a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts +++ b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts @@ -415,15 +415,15 @@ Array [ "group1", ], "rank": "1", - "series_0.data": 0.75, + "series_0.data": 0.6, "ts_bucket": "0", }, Object { "group": Array [ "group2", ], - "rank": "1", - "series_0.data": 0.75, + "rank": "2", + "series_0.data": 0.3, "ts_bucket": "0", }, ] From 32890c38923584914a5f6076982f60b80e392eac Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 7 Jan 2024 16:20:15 -0800 Subject: [PATCH 7/8] test: test count per min agg fn when granularity is specified --- .../clickhouse/__tests__/clickhouse.test.ts | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts index 385526e5d..2c766f61f 100644 --- a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts +++ b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts @@ -323,6 +323,80 @@ Array [ "ts_bucket": 1641341100, }, ] +`); + + const multiGroupBysData2 = ( + await clickhouse.getMultiSeriesChart({ + series: [ + { + type: 'time', + table: 'logs', + aggFn: clickhouse.AggFn.CountPerMin, + field: 'awesomeNumber', + where: `runId:${runId}`, + groupBy: ['testGroup', 'testOtherGroup'], + }, + ], + tableVersion: undefined, + teamId, + startTime: now, + endTime: now + ms('10m'), + granularity: '5 minute', + maxNumGroups: 20, + seriesReturnType: clickhouse.SeriesReturnType.Column, + }) + ).data.map(d => { + return _.pick(d, [ + 'group', + 'series_0.data', + 'series_1.data', + 'ts_bucket', + ]); + }); + expect(multiGroupBysData2.length).toEqual(5); + expect(multiGroupBysData2).toMatchInlineSnapshot(` +Array [ + Object { + "group": Array [ + "group2", + "otherGroup1", + ], + "series_0.data": 0.6, + "ts_bucket": 1641340800, + }, + Object { + "group": Array [ + "group1", + "otherGroup1", + ], + "series_0.data": 0.4, + "ts_bucket": 1641340800, + }, + Object { + "group": Array [ + "group1", + "otherGroup2", + ], + "series_0.data": 0.2, + "ts_bucket": 1641340800, + }, + Object { + "group": Array [ + "group1", + "otherGroup2", + ], + "series_0.data": 0.4, + "ts_bucket": 1641341100, + }, + Object { + "group": Array [ + "group1", + "otherGroup3", + ], + "series_0.data": 0.2, + "ts_bucket": 1641341100, + }, +] `); const ratioData = ( From b3c535a39837dafc07c1cbdfe637dc5e4f6aa010 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 7 Jan 2024 16:21:57 -0800 Subject: [PATCH 8/8] style: cleanup --- packages/api/src/clickhouse/__tests__/clickhouse.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts index 2c766f61f..3e3312381 100644 --- a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts +++ b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts @@ -598,8 +598,6 @@ Array [ }), ); - mockLogsPropertyTypeMappingsModel({}); - mockSpyMetricPropertyTypeMappingsModel({ runId: 'string', host: 'string',