Add postGroupWhere filter option to /chart/series endpoint

.changeset/hip-pans-hear.md

@@ -0,0 +1,5 @@
+---
+'@hyperdx/api': patch
+---
+
+Add postGroupWhere filter option to /chart/series endpoint

packages/api/src/clickhouse/__tests__/clickhouse.test.ts

@@ -1069,6 +1069,77 @@ Array [
       ]
       `);
     });
+
+    it('filters using postGroupWhere properly', async () => {
+      const queryConfig: Parameters<typeof clickhouse.getMultiSeriesChart>[0] =
+        {
+          series: [
+            {
+              type: 'time',
+              table: 'metrics',
+              aggFn: clickhouse.AggFn.LastValue,
+              field: 'test.cpu',
+              where: `runId:${runId}`,
+              groupBy: ['host'],
+              metricDataType: clickhouse.MetricsDataType.Gauge,
+            },
+          ],
+          tableVersion: undefined,
+          teamId,
+          startTime: now,
+          endTime: now + ms('20m'),
+          granularity: undefined,
+          maxNumGroups: 20,
+          seriesReturnType: clickhouse.SeriesReturnType.Column,
+          postGroupWhere: 'series_0:4',
+        };
+
+      // Exclude postGroupWhere to assert we get the test data we expect at first
+      const data = (
+        await clickhouse.getMultiSeriesChart(
+          _.omit(queryConfig, ['postGroupWhere']),
+        )
+      ).data.map(d => {
+        return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
+      });
+
+      expect(data).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "group": Array [
+      "test1",
+    ],
+    "series_0.data": 80,
+    "ts_bucket": "0",
+  },
+  Object {
+    "group": Array [
+      "test2",
+    ],
+    "series_0.data": 4,
+    "ts_bucket": "0",
+  },
+]
+`);
+
+      const filteredData = (
+        await clickhouse.getMultiSeriesChart(queryConfig)
+      ).data.map(d => {
+        return _.pick(d, ['group', 'series_0.data', 'ts_bucket']);
+      });
+
+      expect(filteredData).toMatchInlineSnapshot(`
+Array [
+  Object {
+    "group": Array [
+      "test2",
+    ],
+    "series_0.data": 4,
+    "ts_bucket": "0",
+  },
+]
+`);
+    });
   });
 
   it('limits groups and sorts multi series charts properly', async () => {

packages/api/src/clickhouse/index.ts

@@ -32,6 +32,7 @@ import {
   MetricsPropertyTypeMappingsModel,
 } from './propertyTypeMappingsModel';
 import {
+  buildPostGroupWhereCondition,
   buildSearchColumnName,
   buildSearchColumnName_OLD,
   buildSearchQueryWhereCondition,
@@ -1251,12 +1252,14 @@ export const queryMultiSeriesChart = async ({
   teamId,
   seriesReturnType = SeriesReturnType.Column,
   queries,
+  postGroupWhere,
 }: {
   maxNumGroups: number;
   tableVersion: number | undefined;
   teamId: string;
   seriesReturnType?: SeriesReturnType;
   queries: { query: string; hasGroupBy: boolean; sortOrder?: 'desc' | 'asc' }[];
+  postGroupWhere?: string;
 }) => {
   // For now only supports same-table series with the same groupBy
 
@@ -1296,6 +1299,10 @@ export const queryMultiSeriesChart = async ({
           .join(',\n')
       : 'series_0.data / series_1.data as "series_0.data"';
 
+  const postGroupWhereClause = postGroupWhere
+    ? buildPostGroupWhereCondition({ query: postGroupWhere })
+    : '1=1';
+
   // Return each series data as a separate column
   const query = SqlString.format(
     `WITH ? 
@@ -1306,6 +1313,7 @@ export const queryMultiSeriesChart = async ({
           series_0.group as group
         FROM series_0 AS series_0
         ?
+        WHERE ?
       ), groups AS (
         SELECT *, ?(?) OVER (PARTITION BY group) as rank_order_by_value
         FROM raw_groups
@@ -1323,6 +1331,7 @@ export const queryMultiSeriesChart = async ({
       SqlString.raw(seriesCTEs),
       SqlString.raw(select),
       SqlString.raw(leftJoin),
+      SqlString.raw(postGroupWhereClause),
       // Setting rank_order_by_value
       SqlString.raw(sortOrder === 'asc' ? 'MIN' : 'MAX'),
       SqlString.raw(
@@ -1373,6 +1382,7 @@ export const getMultiSeriesChart = async ({
   tableVersion,
   teamId,
   seriesReturnType = SeriesReturnType.Column,
+  postGroupWhere,
 }: {
   series: z.infer<typeof chartSeriesSchema>[];
   endTime: number; // unix in ms,
@@ -1382,6 +1392,7 @@ export const getMultiSeriesChart = async ({
   tableVersion: number | undefined;
   teamId: string;
   seriesReturnType?: SeriesReturnType;
+  postGroupWhere?: string;
 }) => {
   let queries: {
     query: string;
@@ -1496,6 +1507,7 @@ export const getMultiSeriesChart = async ({
     teamId,
     seriesReturnType,
     queries,
+    postGroupWhere,
   });
 };
 

packages/api/src/clickhouse/searchQueryParser.ts

@@ -652,3 +652,21 @@ export const buildSearchQueryWhereCondition = async ({
   builder.teamId = teamId;
   return await builder.timestampInBetween(startTime, endTime).build();
 };
+
+export const buildPostGroupWhereCondition = ({ query }: { query: string }) => {
+  // This needs to be replaced with the proper query builder
+  // after generalizing it for arbitrary field resolutions
+  // the query can only specify one field from the series and an exact match
+  const [field, value] = query.split(':', 2);
+  const seriesNumber = parseInt(field.replace('series_', ''), 10);
+  const floatValue = parseFloat(value);
+
+  if (Number.isSafeInteger(seriesNumber) === false) {
+    throw new Error('Invalid series number');
+  }
+  if (Number.isNaN(floatValue)) {
+    throw new Error('Invalid value');
+  }
+
+  return SqlString.format(`series_${seriesNumber}.data = ?`, [floatValue]);
+};

packages/api/src/routers/api/chart.ts

@@ -131,13 +131,20 @@ router.post(
       granularity: z.nativeEnum(clickhouse.Granularity).optional(),
       startTime: z.number(),
       seriesReturnType: z.optional(z.nativeEnum(clickhouse.SeriesReturnType)),
+      postGroupWhere: z.optional(z.string().max(1024)),
     }),
   }),
   async (req, res, next) => {
     try {
       const teamId = req.user?.team;
-      const { endTime, granularity, startTime, seriesReturnType, series } =
-        req.body;
+      const {
+        endTime,
+        granularity,
+        startTime,
+        seriesReturnType,
+        series,
+        postGroupWhere,
+      } = req.body;
 
       if (teamId == null) {
         return res.sendStatus(403);
@@ -161,6 +168,7 @@ router.post(
           tableVersion: team.logStreamTableVersion,
           teamId: teamId.toString(),
           seriesReturnType,
+          postGroupWhere,
         }),
       );
     } catch (e) {