From f105fad37a73f86a7dabe9d345f78e69fc2fd5f0 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:15:06 -0800 Subject: [PATCH 1/7] feat: api to pull service linkings --- packages/api/src/fixtures.ts | 3 + .../src/routers/api/__tests__/chart.test.ts | 90 +++++++++++++++++++ packages/api/src/routers/api/chart.ts | 83 ++++++++++++++++- 3 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 packages/api/src/routers/api/__tests__/chart.test.ts diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index 1a7256cb5..fd7440104 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -161,6 +161,7 @@ export function buildEvent({ type = LogType.Log, end_timestamp = 0, span_name, + service = 'test-service', ...properties }: { level?: string; @@ -170,6 +171,7 @@ export function buildEvent({ type?: LogType; end_timestamp?: number; //ms timestamp span_name?: string; + service?: string; } & { [key: string]: number | string | boolean; }): LogStreamModel { @@ -203,6 +205,7 @@ export function buildEvent({ observed_timestamp: `${ts}000000`, _source: source, _platform: platform, + _service: service, severity_text: level, // @ts-ignore end_timestamp: `${end_timestamp}000000`, diff --git a/packages/api/src/routers/api/__tests__/chart.test.ts b/packages/api/src/routers/api/__tests__/chart.test.ts new file mode 100644 index 000000000..cc4c89b78 --- /dev/null +++ b/packages/api/src/routers/api/__tests__/chart.test.ts @@ -0,0 +1,90 @@ +import ms from 'ms'; + +import * as clickhouse from '@/clickhouse'; +import { + buildEvent, + clearClickhouseTables, + clearDBCollections, + clearRedis, + closeDB, + getLoggedInAgent, + getServer, + mockLogsPropertyTypeMappingsModel, +} from '@/fixtures'; + +describe('charts router', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await clearDBCollections(); + await clearClickhouseTables(); + await clearRedis(); + }); + + afterAll(async () => { + await server.closeHttpServer(); + await closeDB(); + }); + + it('GET /chart/services', async () => { + const now = Date.now(); + const { agent, team } = await getLoggedInAgent(server); + + await clickhouse.bulkInsertTeamLogStream( + team.logStreamTableVersion, + team.id, + [ + buildEvent({ + timestamp: now - ms('30m'), + service: 'service1', + 'k8s.namespace.name': 'namespace1', + 'k8s.pod.name': 'pod1', + 'k8s.pod.uid': 'uid1', + }), + buildEvent({ + timestamp: now - ms('1d'), + service: 'service1', + 'k8s.namespace.name': 'namespace1', + 'k8s.pod.name': 'pod2', + 'k8s.pod.uid': 'uid2', + }), + buildEvent({ + timestamp: now - ms('1h'), + service: 'service2', + 'k8s.namespace.name': 'namespace2', + 'k8s.pod.name': 'pod3', + 'k8s.pod.uid': 'uid3', + }), + ], + ); + + const results = await agent.get('/chart/services').expect(200); + expect(results.body.data).toMatchInlineSnapshot(` +Object { + "service1": Array [ + Object { + "k8s.namespace.name": "namespace1", + "k8s.pod.name": "pod1", + "k8s.pod.uid": "uid1", + }, + Object { + "k8s.namespace.name": "namespace1", + "k8s.pod.name": "pod2", + "k8s.pod.uid": "uid2", + }, + ], + "service2": Array [ + Object { + "k8s.namespace.name": "namespace2", + "k8s.pod.name": "pod3", + "k8s.pod.uid": "uid3", + }, + ], +} +`); + }); +}); diff --git a/packages/api/src/routers/api/chart.ts b/packages/api/src/routers/api/chart.ts index 3058ab72e..ec6d409ee 100644 --- a/packages/api/src/routers/api/chart.ts +++ b/packages/api/src/routers/api/chart.ts @@ -1,16 +1,95 @@ import opentelemetry, { SpanStatusCode } from '@opentelemetry/api'; import express from 'express'; -import { isNumber, parseInt } from 'lodash'; +import { isNumber } from 'lodash'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; +import ms from 'ms'; +import { buildSearchColumnName } from '@/clickhouse/searchQueryParser'; import * as clickhouse from '@/clickhouse'; import { getTeam } from '@/controllers/team'; -import logger from '@/utils/logger'; import { chartSeriesSchema } from '@/utils/zod'; const router = express.Router(); +router.get('/services', async (req, res, next) => { + try { + const teamId = req.user?.team; + if (teamId == null) { + return res.sendStatus(403); + } + const team = await getTeam(teamId); + if (team == null) { + return res.sendStatus(403); + } + + const FIELDS = ['k8s.namespace.name', 'k8s.pod.name', 'k8s.pod.uid']; + const nowInMs = Date.now(); + const startTime = nowInMs - ms('5d'); + const endTime = nowInMs; + + const propertyTypeMappingsModel = + await clickhouse.buildLogsPropertyTypeMappingsModel( + team.logStreamTableVersion, + teamId.toString(), + startTime, + endTime, + ); + + const targetGroupByFields: string[] = ['service']; + // make sure all custom fields exist + for (const f of FIELDS) { + if (buildSearchColumnName(propertyTypeMappingsModel.get(f), f)) { + targetGroupByFields.push(f); + } + } + + const MAX_NUM_GROUPS = 50; + + const results = await clickhouse.getMultiSeriesChart({ + series: [ + { + aggFn: clickhouse.AggFn.Count, + groupBy: targetGroupByFields, + table: 'logs', + type: 'table', + where: '', + }, + ], + endTime, + granularity: undefined, + maxNumGroups: MAX_NUM_GROUPS, + startTime, + tableVersion: team.logStreamTableVersion, + teamId: teamId.toString(), + seriesReturnType: clickhouse.SeriesReturnType.Column, + }); + + // restructure service maps + const serviceMap = {}; + for (const row of results.data) { + const values = row.group; + const service = values[0]; + if (!(service in serviceMap)) { + serviceMap[service] = []; + } + const k8sAttrs = {}; + for (let i = 1; i < values.length; i++) { + const field = targetGroupByFields[i]; + const value = values[i]; + k8sAttrs[field] = value; + } + serviceMap[service].push(k8sAttrs); + } + + res.json({ + data: serviceMap, + }); + } catch (e) { + next(e); + } +}); + router.post( '/series', validateRequest({ From 64eee5ca745a98d91704e862b1e938e35bd3b0d4 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:41:43 -0800 Subject: [PATCH 2/7] perf: add caching --- packages/api/src/routers/api/chart.ts | 50 +++++++++++++++------------ 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/packages/api/src/routers/api/chart.ts b/packages/api/src/routers/api/chart.ts index ec6d409ee..1133eeabb 100644 --- a/packages/api/src/routers/api/chart.ts +++ b/packages/api/src/routers/api/chart.ts @@ -1,13 +1,14 @@ import opentelemetry, { SpanStatusCode } from '@opentelemetry/api'; import express from 'express'; import { isNumber } from 'lodash'; +import ms from 'ms'; import { z } from 'zod'; import { validateRequest } from 'zod-express-middleware'; -import ms from 'ms'; -import { buildSearchColumnName } from '@/clickhouse/searchQueryParser'; import * as clickhouse from '@/clickhouse'; +import { buildSearchColumnName } from '@/clickhouse/searchQueryParser'; import { getTeam } from '@/controllers/team'; +import { SimpleCache } from '@/utils/redis'; import { chartSeriesSchema } from '@/utils/zod'; const router = express.Router(); @@ -46,34 +47,39 @@ router.get('/services', async (req, res, next) => { const MAX_NUM_GROUPS = 50; - const results = await clickhouse.getMultiSeriesChart({ - series: [ - { - aggFn: clickhouse.AggFn.Count, - groupBy: targetGroupByFields, - table: 'logs', - type: 'table', - where: '', - }, - ], - endTime, - granularity: undefined, - maxNumGroups: MAX_NUM_GROUPS, - startTime, - tableVersion: team.logStreamTableVersion, - teamId: teamId.toString(), - seriesReturnType: clickhouse.SeriesReturnType.Column, - }); + const simpleCache = new SimpleCache< + Awaited> + >(`chart-services-${teamId}`, ms('10m'), () => + clickhouse.getMultiSeriesChart({ + series: [ + { + aggFn: clickhouse.AggFn.Count, + groupBy: targetGroupByFields, + table: 'logs', + type: 'table', + where: '', + }, + ], + endTime, + granularity: undefined, + maxNumGroups: MAX_NUM_GROUPS, + startTime, + tableVersion: team.logStreamTableVersion, + teamId: teamId.toString(), + seriesReturnType: clickhouse.SeriesReturnType.Column, + }), + ); + const results = await simpleCache.get(); // restructure service maps - const serviceMap = {}; + const serviceMap: Record[]> = {}; for (const row of results.data) { const values = row.group; const service = values[0]; if (!(service in serviceMap)) { serviceMap[service] = []; } - const k8sAttrs = {}; + const k8sAttrs: Record = {}; for (let i = 1; i < values.length; i++) { const field = targetGroupByFields[i]; const value = values[i]; From 4225d33087b08414b17076a32c5bbaf972f8714e Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:50:09 -0800 Subject: [PATCH 3/7] fix: bump max groups --- packages/api/src/routers/api/chart.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/routers/api/chart.ts b/packages/api/src/routers/api/chart.ts index 1133eeabb..00b83ee3c 100644 --- a/packages/api/src/routers/api/chart.ts +++ b/packages/api/src/routers/api/chart.ts @@ -45,7 +45,7 @@ router.get('/services', async (req, res, next) => { } } - const MAX_NUM_GROUPS = 50; + const MAX_NUM_GROUPS = 2000; const simpleCache = new SimpleCache< Awaited> From 1cb172f0625ecbd6a3b7fc62761f2f09a5d6eee7 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:51:26 -0800 Subject: [PATCH 4/7] docs: add changeset --- .changeset/serious-experts-whisper.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/serious-experts-whisper.md diff --git a/.changeset/serious-experts-whisper.md b/.changeset/serious-experts-whisper.md new file mode 100644 index 000000000..5c70417ed --- /dev/null +++ b/.changeset/serious-experts-whisper.md @@ -0,0 +1,6 @@ +--- +'@hyperdx/api': patch +'@hyperdx/app': patch +--- + +feat: api to pull service + k8s attrs linkings From fe95f38fbd24c3631cb5870c17089be71771981b Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:54:26 -0800 Subject: [PATCH 5/7] test: missing attributes --- .../src/routers/api/__tests__/chart.test.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/packages/api/src/routers/api/__tests__/chart.test.ts b/packages/api/src/routers/api/__tests__/chart.test.ts index cc4c89b78..2905a2a2c 100644 --- a/packages/api/src/routers/api/__tests__/chart.test.ts +++ b/packages/api/src/routers/api/__tests__/chart.test.ts @@ -85,6 +85,58 @@ Object { }, ], } +`); + }); + + it('GET /chart/services (missing custom attributes)', async () => { + const now = Date.now(); + const { agent, team } = await getLoggedInAgent(server); + + await clickhouse.bulkInsertTeamLogStream( + team.logStreamTableVersion, + team.id, + [ + buildEvent({ + timestamp: now - ms('30m'), + service: 'service1', + 'k8s.namespace.name': 'namespace1', + 'k8s.pod.uid': 'uid1', + }), + buildEvent({ + timestamp: now - ms('1d'), + service: 'service1', + 'k8s.namespace.name': 'namespace1', + 'k8s.pod.uid': 'uid2', + }), + buildEvent({ + timestamp: now - ms('1h'), + service: 'service2', + 'k8s.namespace.name': 'namespace2', + 'k8s.pod.uid': 'uid3', + }), + ], + ); + + const results = await agent.get('/chart/services').expect(200); + expect(results.body.data).toMatchInlineSnapshot(` +Object { + "service1": Array [ + Object { + "k8s.namespace.name": "namespace1", + "k8s.pod.uid": "uid1", + }, + Object { + "k8s.namespace.name": "namespace1", + "k8s.pod.uid": "uid2", + }, + ], + "service2": Array [ + Object { + "k8s.namespace.name": "namespace2", + "k8s.pod.uid": "uid3", + }, + ], +} `); }); }); From a38a2bbaf4820639a7e9c654992f80597c4dbdb4 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:57:10 -0800 Subject: [PATCH 6/7] test: cleanups --- .../src/routers/api/__tests__/chart.test.ts | 33 +++++-------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/packages/api/src/routers/api/__tests__/chart.test.ts b/packages/api/src/routers/api/__tests__/chart.test.ts index 2905a2a2c..c68871b00 100644 --- a/packages/api/src/routers/api/__tests__/chart.test.ts +++ b/packages/api/src/routers/api/__tests__/chart.test.ts @@ -9,7 +9,6 @@ import { closeDB, getLoggedInAgent, getServer, - mockLogsPropertyTypeMappingsModel, } from '@/fixtures'; describe('charts router', () => { @@ -39,21 +38,21 @@ describe('charts router', () => { team.id, [ buildEvent({ - timestamp: now - ms('30m'), + timestamp: now, service: 'service1', 'k8s.namespace.name': 'namespace1', 'k8s.pod.name': 'pod1', 'k8s.pod.uid': 'uid1', }), buildEvent({ - timestamp: now - ms('1d'), + timestamp: now, service: 'service1', 'k8s.namespace.name': 'namespace1', 'k8s.pod.name': 'pod2', 'k8s.pod.uid': 'uid2', }), buildEvent({ - timestamp: now - ms('1h'), + timestamp: now - ms('1d'), service: 'service2', 'k8s.namespace.name': 'namespace2', 'k8s.pod.name': 'pod3', @@ -97,22 +96,16 @@ Object { team.id, [ buildEvent({ - timestamp: now - ms('30m'), + timestamp: now, service: 'service1', - 'k8s.namespace.name': 'namespace1', - 'k8s.pod.uid': 'uid1', }), buildEvent({ - timestamp: now - ms('1d'), + timestamp: now, service: 'service1', - 'k8s.namespace.name': 'namespace1', - 'k8s.pod.uid': 'uid2', }), buildEvent({ - timestamp: now - ms('1h'), + timestamp: now - ms('1d'), service: 'service2', - 'k8s.namespace.name': 'namespace2', - 'k8s.pod.uid': 'uid3', }), ], ); @@ -121,20 +114,10 @@ Object { expect(results.body.data).toMatchInlineSnapshot(` Object { "service1": Array [ - Object { - "k8s.namespace.name": "namespace1", - "k8s.pod.uid": "uid1", - }, - Object { - "k8s.namespace.name": "namespace1", - "k8s.pod.uid": "uid2", - }, + Object {}, ], "service2": Array [ - Object { - "k8s.namespace.name": "namespace2", - "k8s.pod.uid": "uid3", - }, + Object {}, ], } `); From 83d59daf025f730dc29e147963e17b955ff1521d Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 5 Jan 2024 14:00:13 -0800 Subject: [PATCH 7/7] fix: test --- packages/api/src/clickhouse/__tests__/clickhouse.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts index 8e9bc54bd..1b56f1aa3 100644 --- a/packages/api/src/clickhouse/__tests__/clickhouse.test.ts +++ b/packages/api/src/clickhouse/__tests__/clickhouse.test.ts @@ -81,7 +81,7 @@ Array [ Object { "_host": "", "_platform": "nodejs", - "_service": "", + "_service": "test-service", "body": "", "duration": -1641340800001, "severity_text": "", @@ -92,7 +92,7 @@ Array [ Object { "_host": "", "_platform": "nodejs", - "_service": "", + "_service": "test-service", "body": "", "duration": -1641340800000, "severity_text": "",