Skip to content

Commit

Permalink
feat: implement histogram linear interpolation quantile function (#253)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wrn14897
wrn14897 authored Jan 24, 2024
1 parent 52d9b71 commit 82640b0
Show file tree
Hide file tree
Showing 5 changed files with 1,353 additions and 332 deletions.
6 changes: 6 additions & 0 deletions .changeset/bright-dancers-float.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@hyperdx/api': patch
'@hyperdx/app': patch
---

feat: implement histogram linear interpolation quantile function
319 changes: 0 additions & 319 deletions packages/api/src/clickhouse/__tests__/clickhouse.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -871,325 +871,6 @@ Array [
`);
});

describe('fetches multi series metric table chart correctly', () => {
const now = new Date('2022-01-05').getTime();
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = `test`;

beforeEach(async () => {
// Rate: 8, 1, 8, 25
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.users',
tags: { host: 'test1', runId, ip: '127.0.0.1' },
data_type: clickhouse.MetricsDataType.Sum,
is_monotonic: true,
is_delta: true,
unit: 'Users',
points: [
{ value: 0, timestamp: now - ms('1m') }, // 0
{ value: 1, timestamp: now },
{ value: 8, timestamp: now + ms('4m') }, // 8
{ value: 8, timestamp: now + ms('6m') },
{ value: 9, timestamp: now + ms('9m') }, // 9
{ value: 15, timestamp: now + ms('11m') },
{ value: 17, timestamp: now + ms('14m') }, // 17
{ value: 32, timestamp: now + ms('16m') },
{ value: 42, timestamp: now + ms('19m') }, // 42
],
}),
);

// Rate: 11, 78, 5805, 78729
// Sum: 12, 79, 5813, 78754
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.users',
tags: { host: 'test2', runId, ip: '127.0.0.2' },
data_type: clickhouse.MetricsDataType.Sum,
is_monotonic: true,
is_delta: true,
unit: 'Users',
points: [
{ value: 3, timestamp: now - ms('1m') }, // 3
{ value: 3, timestamp: now },
{ value: 14, timestamp: now + ms('4m') }, // 14
{ value: 15, timestamp: now + ms('6m') },
{ value: 92, timestamp: now + ms('9m') }, // 92
{ value: 653, timestamp: now + ms('11m') },
{ value: 5897, timestamp: now + ms('14m') }, // 5897
{ value: 9323, timestamp: now + ms('16m') },
{ value: 84626, timestamp: now + ms('19m') }, // 84626
],
}),
);

await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.cpu',
tags: { host: 'test1', runId, ip: '127.0.0.1' },
data_type: clickhouse.MetricsDataType.Gauge,
is_monotonic: false,
is_delta: false,
unit: 'Percent',
points: [
{ value: 50, timestamp: now },
{ value: 25, timestamp: now + ms('1m') },
{ value: 12.5, timestamp: now + ms('2m') },
{ value: 6.25, timestamp: now + ms('3m') }, // Last 5min
{ value: 100, timestamp: now + ms('6m') },
{ value: 75, timestamp: now + ms('7m') },
{ value: 10, timestamp: now + ms('8m') },
{ value: 80, timestamp: now + ms('9m') }, // Last 5min
],
}),
);

await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.cpu',
tags: { host: 'test2', runId, ip: '127.0.0.2' },
data_type: clickhouse.MetricsDataType.Gauge,
is_monotonic: false,
is_delta: false,
unit: 'Percent',
points: [
{ value: 1, timestamp: now },
{ value: 2, timestamp: now + ms('1m') },
{ value: 3, timestamp: now + ms('2m') },
{ value: 4, timestamp: now + ms('3m') }, // Last 5min
{ value: 5, timestamp: now + ms('6m') },
{ value: 6, timestamp: now + ms('7m') },
{ value: 5, timestamp: now + ms('8m') },
{ value: 4, timestamp: now + ms('9m') }, // Last 5min
],
}),
);

mockSpyMetricPropertyTypeMappingsModel({
runId: 'string',
host: 'string',
});
});

it('returns multiple group by labels correctly', async () => {
const data = (
await clickhouse.getMultiSeriesChart({
series: [
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.LastValue,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: ['host', 'ip'],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
})
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test1",
"127.0.0.1",
],
"series_0.data": 80,
"ts_bucket": "0",
},
Object {
"group": Array [
"test2",
"127.0.0.2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);
});

it('gauge (last value)', async () => {
const data = (
await clickhouse.getMultiSeriesChart({
series: [
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.LastValue,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: ['host'],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
})
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test1",
],
"series_0.data": 80,
"ts_bucket": "0",
},
Object {
"group": Array [
"test2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);
});

it('sum (rate) + gauge (avg)', async () => {
const data = (
await clickhouse.getMultiSeriesChart({
series: [
{
type: 'table',
table: 'metrics',
aggFn: clickhouse.AggFn.SumRate,
field: 'test.users',
where: `runId:${runId}`,
groupBy: [],
metricDataType: clickhouse.MetricsDataType.Sum,
},
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.Avg,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: [],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
})
).data.map(d => {
return _.pick(d, [
'group',
'series_0.data',
'series_1.data',
'ts_bucket',
]);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [],
"series_0.data": 84665,
"series_1.data": 42,
"ts_bucket": "0",
},
]
`);
});

it('filters using postGroupWhere properly', async () => {
const queryConfig: Parameters<typeof clickhouse.getMultiSeriesChart>[0] =
{
series: [
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.LastValue,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: ['host'],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
postGroupWhere: 'series_0:4',
};

// Exclude postGroupWhere to assert we get the test data we expect at first
const data = (
await clickhouse.getMultiSeriesChart(
_.omit(queryConfig, ['postGroupWhere']),
)
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test1",
],
"series_0.data": 80,
"ts_bucket": "0",
},
Object {
"group": Array [
"test2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);

const filteredData = (
await clickhouse.getMultiSeriesChart(queryConfig)
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(filteredData).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);
});
});

it('limits groups and sorts multi series charts properly', async () => {
const now = new Date('2022-01-05').getTime();
const runId = Math.random().toString(); // dedup watch mode runs
Expand Down
Loading

0 comments on commit 82640b0

Please sign in to comment.