Merge branch 'main' into mikeshi/support-escaped-quotes

.changeset/chilled-insects-boil.md

@@ -0,0 +1,6 @@
+---
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+feat: introduce go-parser service

.changeset/violet-singers-melt.md

@@ -0,0 +1,6 @@
+---
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+fix: services endpoint bug (missing log lines results in no matches)

.changeset/wild-plants-drop.md

@@ -0,0 +1,6 @@
+---
+'@hyperdx/api': patch
+'@hyperdx/app': patch
+---
+
+fix: GET alerts endpoint

docker-compose.dev.yml

@@ -1,5 +1,22 @@
 version: '3'
 services:
+  go-parser:
+    container_name: hdx-oss-dev-go-parser
+    build:
+      context: .
+      dockerfile: ./packages/go-parser/Dockerfile
+    environment:
+      AGGREGATOR_API_URL: 'http://aggregator:8001'
+      HYPERDX_API_KEY: ${HYPERDX_API_KEY}
+      HYPERDX_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
+      OTEL_LOG_LEVEL: ${HYPERDX_LOG_LEVEL}
+      OTEL_SERVICE_NAME: hdx-oss-dev-go-parser
+      PORT: 7777
+    ports:
+      - 7777:7777
+    networks:
+      - internal
   miner:
     container_name: hdx-oss-dev-miner
     build:
@@ -45,6 +62,7 @@ services:
       - 8002:8002 # http-generic
       - 8686:8686 # healthcheck
     environment:
+      ENABLE_GO_PARSER: 'false'
       RUST_BACKTRACE: full
       VECTOR_LOG: ${HYPERDX_LOG_LEVEL}
       VECTOR_OPENSSL_LEGACY_PROVIDER: 'false'

docker/ingestor/core.toml

@@ -706,6 +706,20 @@ if is_object(.r) {
   del(.hdx_content_type)
 }
 '''
+
+[transforms.post_spans]
+type = "filter"
+inputs = ["spans"]
+condition = '''
+  ("${ENABLE_GO_PARSER:-false}" == "true" && is_nullish(.b."db.statement")) || "${ENABLE_GO_PARSER:-false}" == "false"
+'''
+
+[transforms.go_spans]
+type = "filter"
+inputs = ["spans"]
+condition = '''
+  "${ENABLE_GO_PARSER:-false}" == "true" && !is_nullish(.b."db.statement")
+'''
 # --------------------------------------------------------------------------------
 
 

docker/ingestor/http-sinks.toml

@@ -1,7 +1,18 @@
+[sinks.go_parser]
+type = "http"
+uri = "http://go-parser:7777"
+inputs = ["go_spans"] # only send spans for now
+compression = "gzip"
+encoding.codec = "json"
+batch.max_bytes = 10485760 # 10MB, required for rrweb payloads 
+batch.max_events = 100
+batch.timeout_secs = 1
+
+
 [sinks.dev_hdx_aggregator]
 type = "http"
 uri = "http://aggregator:8001"
-inputs = ["spans", "post_logs"]
+inputs = ["post_spans", "post_logs"]
 compression = "gzip"
 encoding.codec = "json"
 batch.max_bytes = 10485760 # 10MB, required for rrweb payloads 

packages/api/src/controllers/alerts.ts

@@ -3,6 +3,7 @@ import ms from 'ms';
 
 import * as clickhouse from '@/clickhouse';
 import { SQLSerializer } from '@/clickhouse/searchQueryParser';
+import type { ObjectId } from '@/models';
 import Alert, {
   AlertChannel,
   AlertInterval,
@@ -76,11 +77,11 @@ export const validateGroupByProperty = async ({
 
 const makeAlert = (alert: AlertInput) => {
   return {
-    source: alert.source,
     channel: alert.channel,
     interval: alert.interval,
-    type: alert.type,
+    source: alert.source,
     threshold: alert.threshold,
+    type: alert.type,
     // Log alerts
     logView: alert.logViewId,
     groupBy: alert.groupBy,
@@ -92,12 +93,20 @@ const makeAlert = (alert: AlertInput) => {
   };
 };
 
-export const createAlert = async (alertInput: AlertInput) => {
-  return new Alert(makeAlert(alertInput)).save();
+export const createAlert = async (teamId: ObjectId, alertInput: AlertInput) => {
+  return new Alert({
+    ...makeAlert(alertInput),
+    team: teamId,
+  }).save();
 };
 
 // create an update alert function based off of the above create alert function
-export const updateAlert = async (id: string, alertInput: AlertInput) => {
+export const updateAlert = async (
+  id: string,
+  teamId: ObjectId,
+  alertInput: AlertInput,
+) => {
+  // TODO: find by id and teamId
   // should consider clearing AlertHistory when updating an alert?
   return Alert.findByIdAndUpdate(id, makeAlert(alertInput), {
     returnDocument: 'after',

packages/api/src/models/alert.ts

@@ -34,11 +34,12 @@ export interface IAlert {
   channel: AlertChannel;
   cron: string;
   interval: AlertInterval;
+  source?: AlertSource;
   state: AlertState;
+  team: ObjectId;
   threshold: number;
   timezone: string;
   type: AlertType;
-  source?: AlertSource;
 
   // Log alerts
   groupBy?: string;
@@ -85,6 +86,10 @@ const AlertSchema = new Schema<IAlert>(
       required: false,
       default: 'LOG',
     },
+    team: {
+      type: mongoose.Schema.Types.ObjectId,
+      ref: 'Team',
+    },
 
     // Log alerts
     logView: {

packages/api/src/routers/api/__tests__/chart.test.ts

@@ -9,6 +9,7 @@ import {
   closeDB,
   getLoggedInAgent,
   getServer,
+  mockLogsPropertyTypeMappingsModel,
 } from '@/fixtures';
 
 describe('charts router', () => {
@@ -119,6 +120,43 @@ Object {
   "service1": Array [],
   "service2": Array [],
 }
+`);
+  });
+
+  it('GET /chart/services (missing data but custom attributes exist)', async () => {
+    const now = Date.now();
+    const { agent, team } = await getLoggedInAgent(server);
+
+    await clickhouse.bulkInsertTeamLogStream(
+      team.logStreamTableVersion,
+      team.id,
+      [
+        buildEvent({
+          timestamp: now,
+          service: 'service1',
+        }),
+        buildEvent({
+          timestamp: now,
+          service: 'service1',
+        }),
+        buildEvent({
+          timestamp: now - ms('1d'),
+          service: 'service2',
+        }),
+      ],
+    );
+
+    mockLogsPropertyTypeMappingsModel({
+      service: 'string',
+      'k8s.namespace.name': 'string',
+    });
+
+    const results = await agent.get('/chart/services').expect(200);
+    expect(results.body.data).toMatchInlineSnapshot(`
+Object {
+  "service1": Array [],
+  "service2": Array [],
+}
 `);
   });
 });