Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: implement histogram linear interpolation quantile function #253

Merged
merged 19 commits into from
Jan 24, 2024
Merged

feat: implement histogram linear interpolation quantile function #253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
b194bfd
feat: implement linear interpolation in sql to compute histogram
wrn14897 Jan 19, 2024
b63cf29
test: histogram
wrn14897 Jan 19, 2024
97994d5
Merge branch 'main' into warren/implement-histogram-linear-interpolat…
wrn14897 Jan 19, 2024
859d1aa
fix + perf: consider time dimension + reuse pre-compute length
wrn14897 Jan 19, 2024
1328383
Merge branch 'main' into warren/implement-histogram-linear-interpolat…
wrn14897 Jan 19, 2024
25d6dfa
docs: add changeset
wrn14897 Jan 19, 2024
5037a99
test: histogram granularity
wrn14897 Jan 19, 2024
9e0f375
feedback: handle finite/infinite upper/lower boundaries + invalid point
wrn14897 Jan 19, 2024
01a6fe2
fix: handle 0 lower boundary
wrn14897 Jan 19, 2024
93e841b
test: test getMultiSeriesChart in a separate file
wrn14897 Jan 19, 2024
03038c4
fix: neighbor ordering issue
wrn14897 Jan 20, 2024
63d4b4d
fix: start time offset + rule out the cases if prev points not exist
wrn14897 Jan 20, 2024
33eed74
Merge branch 'main' into warren/implement-histogram-linear-interpolat…
wrn14897 Jan 20, 2024
ee0b37f
Merge branch 'main' into warren/implement-histogram-linear-interpolat…
wrn14897 Jan 23, 2024
93279c9
feat: handle group-by + partial counter reset detection
wrn14897 Jan 23, 2024
efb5157
feat: handle counter reset
wrn14897 Jan 23, 2024
4aab77c
docs: add comment
wrn14897 Jan 23, 2024
7a186d3
fix: reuse existing rateMetricSource to handle counter reset
wrn14897 Jan 24, 2024
b2a0d63
test: counter might reset
wrn14897 Jan 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/bright-dancers-float.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@hyperdx/api': patch
'@hyperdx/app': patch
---

feat: implement histogram linear interpolation quantile function
319 changes: 0 additions & 319 deletions packages/api/src/clickhouse/__tests__/clickhouse.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -871,325 +871,6 @@ Array [
`);
});

describe('fetches multi series metric table chart correctly', () => {
const now = new Date('2022-01-05').getTime();
const runId = Math.random().toString(); // dedup watch mode runs
const teamId = `test`;

beforeEach(async () => {
// Rate: 8, 1, 8, 25
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.users',
tags: { host: 'test1', runId, ip: '127.0.0.1' },
data_type: clickhouse.MetricsDataType.Sum,
is_monotonic: true,
is_delta: true,
unit: 'Users',
points: [
{ value: 0, timestamp: now - ms('1m') }, // 0
{ value: 1, timestamp: now },
{ value: 8, timestamp: now + ms('4m') }, // 8
{ value: 8, timestamp: now + ms('6m') },
{ value: 9, timestamp: now + ms('9m') }, // 9
{ value: 15, timestamp: now + ms('11m') },
{ value: 17, timestamp: now + ms('14m') }, // 17
{ value: 32, timestamp: now + ms('16m') },
{ value: 42, timestamp: now + ms('19m') }, // 42
],
}),
);

// Rate: 11, 78, 5805, 78729
// Sum: 12, 79, 5813, 78754
await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.users',
tags: { host: 'test2', runId, ip: '127.0.0.2' },
data_type: clickhouse.MetricsDataType.Sum,
is_monotonic: true,
is_delta: true,
unit: 'Users',
points: [
{ value: 3, timestamp: now - ms('1m') }, // 3
{ value: 3, timestamp: now },
{ value: 14, timestamp: now + ms('4m') }, // 14
{ value: 15, timestamp: now + ms('6m') },
{ value: 92, timestamp: now + ms('9m') }, // 92
{ value: 653, timestamp: now + ms('11m') },
{ value: 5897, timestamp: now + ms('14m') }, // 5897
{ value: 9323, timestamp: now + ms('16m') },
{ value: 84626, timestamp: now + ms('19m') }, // 84626
],
}),
);

await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.cpu',
tags: { host: 'test1', runId, ip: '127.0.0.1' },
data_type: clickhouse.MetricsDataType.Gauge,
is_monotonic: false,
is_delta: false,
unit: 'Percent',
points: [
{ value: 50, timestamp: now },
{ value: 25, timestamp: now + ms('1m') },
{ value: 12.5, timestamp: now + ms('2m') },
{ value: 6.25, timestamp: now + ms('3m') }, // Last 5min
{ value: 100, timestamp: now + ms('6m') },
{ value: 75, timestamp: now + ms('7m') },
{ value: 10, timestamp: now + ms('8m') },
{ value: 80, timestamp: now + ms('9m') }, // Last 5min
],
}),
);

await clickhouse.bulkInsertTeamMetricStream(
buildMetricSeries({
name: 'test.cpu',
tags: { host: 'test2', runId, ip: '127.0.0.2' },
data_type: clickhouse.MetricsDataType.Gauge,
is_monotonic: false,
is_delta: false,
unit: 'Percent',
points: [
{ value: 1, timestamp: now },
{ value: 2, timestamp: now + ms('1m') },
{ value: 3, timestamp: now + ms('2m') },
{ value: 4, timestamp: now + ms('3m') }, // Last 5min
{ value: 5, timestamp: now + ms('6m') },
{ value: 6, timestamp: now + ms('7m') },
{ value: 5, timestamp: now + ms('8m') },
{ value: 4, timestamp: now + ms('9m') }, // Last 5min
],
}),
);

mockSpyMetricPropertyTypeMappingsModel({
runId: 'string',
host: 'string',
});
});

it('returns multiple group by labels correctly', async () => {
const data = (
await clickhouse.getMultiSeriesChart({
series: [
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.LastValue,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: ['host', 'ip'],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
})
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test1",
"127.0.0.1",
],
"series_0.data": 80,
"ts_bucket": "0",
},
Object {
"group": Array [
"test2",
"127.0.0.2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);
});

it('gauge (last value)', async () => {
const data = (
await clickhouse.getMultiSeriesChart({
series: [
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.LastValue,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: ['host'],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
})
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test1",
],
"series_0.data": 80,
"ts_bucket": "0",
},
Object {
"group": Array [
"test2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);
});

it('sum (rate) + gauge (avg)', async () => {
const data = (
await clickhouse.getMultiSeriesChart({
series: [
{
type: 'table',
table: 'metrics',
aggFn: clickhouse.AggFn.SumRate,
field: 'test.users',
where: `runId:${runId}`,
groupBy: [],
metricDataType: clickhouse.MetricsDataType.Sum,
},
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.Avg,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: [],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
})
).data.map(d => {
return _.pick(d, [
'group',
'series_0.data',
'series_1.data',
'ts_bucket',
]);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [],
"series_0.data": 84665,
"series_1.data": 42,
"ts_bucket": "0",
},
]
`);
});

it('filters using postGroupWhere properly', async () => {
const queryConfig: Parameters<typeof clickhouse.getMultiSeriesChart>[0] =
{
series: [
{
type: 'time',
table: 'metrics',
aggFn: clickhouse.AggFn.LastValue,
field: 'test.cpu',
where: `runId:${runId}`,
groupBy: ['host'],
metricDataType: clickhouse.MetricsDataType.Gauge,
},
],
tableVersion: undefined,
teamId,
startTime: now,
endTime: now + ms('20m'),
granularity: undefined,
maxNumGroups: 20,
seriesReturnType: clickhouse.SeriesReturnType.Column,
postGroupWhere: 'series_0:4',
};

// Exclude postGroupWhere to assert we get the test data we expect at first
const data = (
await clickhouse.getMultiSeriesChart(
_.omit(queryConfig, ['postGroupWhere']),
)
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(data).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test1",
],
"series_0.data": 80,
"ts_bucket": "0",
},
Object {
"group": Array [
"test2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);

const filteredData = (
await clickhouse.getMultiSeriesChart(queryConfig)
).data.map(d => {
return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
});

expect(filteredData).toMatchInlineSnapshot(`
Array [
Object {
"group": Array [
"test2",
],
"series_0.data": 4,
"ts_bucket": "0",
},
]
`);
});
});

it('limits groups and sorts multi series charts properly', async () => {
const now = new Date('2022-01-05').getTime();
const runId = Math.random().toString(); // dedup watch mode runs
Expand Down
Loading
Loading