feat: Exemplars for metric & PromQL charts

.changeset/exemplar-mode-metrics.md

@@ -0,0 +1,33 @@
+---
+"@hyperdx/common-utils": minor
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+---
+
+feat: add exemplar overlay for metric and PromQL charts
+
+Time charts on metric and PromQL sources can now overlay exemplars —
+individual data points linked to a trace — via the "Exemplars" toggle in the
+chart editor (next to "As Ratio" for metric charts, in the PromQL editor for
+PromQL charts). Each marker sits at the trace's own measurement rather than on
+the series line, so its height matches what the linked trace reports; hovering a
+marker shows trace metadata (service, span, duration, status) from a
+configurable exemplar trace source, with a button to open the trace directly.
+
+Markers are sampled the way Grafana samples exemplars: bucketed at the chart's
+granularity, keeping the slowest trace in each bucket plus any further trace
+more than 2σ below it, with the marker budget spread evenly across the time
+range. The overlay therefore shows the shape of the latency distribution
+instead of tracing the top of the chart.
+
+For structured metric sources, exemplars are read directly from the OTel metric
+tables' `Exemplars.*` columns (`renderMetricExemplarsChartConfig`), honoring the
+chart's time range, metric name, and filters. For PromQL sources backed by a
+real Prometheus endpoint, the new `/v1/prometheus/query_exemplars` route proxies
+to Prometheus's native `/api/v1/query_exemplars`. The overlay is opt-in and runs
+its query in parallel only when enabled, so charts that don't use it are
+unaffected. Trace-source exemplar generation lands in a follow-up.
+
+The overlay is off by default for the whole deployment and is enabled with
+`NEXT_PUBLIC_ENABLE_EXEMPLARS=true`; with it unset, the chart-editor toggle is
+hidden and no exemplar query runs even for charts that have the flag saved.

.changeset/span-metrics-connector.md

@@ -0,0 +1,23 @@
+---
+"@hyperdx/api": minor
+"@hyperdx/otel-collector": minor
+---
+
+feat: optional spanmetrics connector for metric exemplars
+
+Adds the `spanmetricsconnector` to the collector build and wires it into the
+OpAMP-generated collector config, gated on the `ENABLE_SPAN_METRICS` env flag
+(off by default). When enabled, the collector derives `traces.span.metrics.*`
+(calls + duration histogram) from spans with **exemplars enabled**, so the
+duration histogram lands in ClickHouse with `Exemplars.*` pointing back at the
+spans they were measured from — giving coherent, fully-OTLP metric exemplars
+without any direct ClickHouse writes. Enabled in local dev to back the new
+`telemetry-generator` service.
+
+The duration histogram uses exponential (OTLP) buckets rather than a fixed
+ladder, so it lands in `otel_metrics_exponential_histogram` and remote-writes
+to Prometheus as a native histogram. A fixed ladder's wide top bucket makes
+high quantiles interpolate well past the slowest request that actually
+happened — a p99 reading 10s off a 5s–10s bucket when nothing took longer than
+6s — which no exemplar can ever sit on. Exemplar overlays now accept
+exponential histograms alongside explicit-bucket ones.

.gitignore

@@ -95,3 +95,7 @@ docker-compose.prod.yml
 
 # webstorm
 .idea
+
+# Stryker mutation-testing output
+packages/*/reports/
+packages/*/.stryker-tmp/

docker-compose.dev.yml

@@ -41,6 +41,9 @@ services:
       OTEL_SUPERVISOR_LOGS: 'true'
       HYPERDX_OTEL_EXPORTER_TABLES_TTL: '24h'
       ENABLE_PROMQL: 'true'
+      # Note: SPAN_METRICS_PROM_RW_ENDPOINT belongs to the API, not here — the
+      # API resolves it and inlines it into the generated collector config. It
+      # lives in packages/api/.env.development.
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
@@ -157,9 +160,17 @@ services:
       hdx.dev.service: prometheus
       hdx.dev.port: '${HDX_DEV_PROMETHEUS_PORT:-9090}'
       hdx.dev.url: 'http://localhost:${HDX_DEV_PROMETHEUS_PORT:-9090}'
-    profiles:
-      - prometheus
     image: prom/prometheus:latest
+    # exemplar-storage: query_exemplars data; remote-write receiver: ingest the
+    # collector's span-derived metrics (with exemplars) from real generated data.
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      # native-histograms: the collector remote-writes the span-derived metrics
+      # as native histograms; without the flag Prometheus drops those samples.
+      - '--enable-feature=exemplar-storage,native-histograms'
+      - '--web.enable-remote-write-receiver'
+      - '--web.enable-lifecycle'
     ports:
       - '${HDX_DEV_PROMETHEUS_PORT:-9090}:9090'
     volumes:
@@ -169,5 +180,25 @@ services:
       - internal
     restart: on-failure
 
+  # Synthetic traces (via OTLP) + coherent metric exemplars (to ClickHouse) for
+  # local dev + e2e. See telemetry-generator/README.md.
+  telemetry-generator:
+    labels:
+      <<: *hdx-labels
+      hdx.dev.service: telemetry-generator
+    build:
+      context: ./telemetry-generator
+    environment:
+      OTEL_EXPORTER_OTLP_ENDPOINT: 'http://otel-collector:4318'
+      GEN_OTLP_API_KEY: '${INGESTION_API_KEY:-super-secure-ingestion-api-key}'
+      GEN_BACKFILL_MINUTES: '30'
+      GEN_RATE_PER_SEC: '20'
+    networks:
+      - internal
+    restart: on-failure
+    depends_on:
+      otel-collector:
+        condition: service_started
+
 networks:
   internal:

docker-compose.yml

@@ -74,6 +74,8 @@ services:
       # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics)
       # ENABLE_PROMQL: 'true'
       # NEXT_PUBLIC_ENABLE_PROMQL: 'true'
+      # Uncomment to enable exemplar overlays (trace markers on time charts)
+      # NEXT_PUBLIC_ENABLE_EXEMPLARS: 'true'
       DEFAULT_CONNECTIONS:
         '[{"name":"Local
         ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]'

docker/prometheus/prometheus.yml

@@ -18,3 +18,7 @@ scrape_configs:
     static_configs:
       - targets: ['ch-server:9363']
     metrics_path: '/metrics'
+
+  # Note: the span-derived request metrics (with exemplars) arrive via
+  # remote-write from the OTel collector, not a scrape — see the collector's
+  # SPAN_METRICS_PROM_RW_ENDPOINT and the spanmetrics connector.

knip.json

@@ -39,6 +39,7 @@
     ".github/scripts/**",
     ".github/actions/**",
     "docker/hyperdx/**",
+    "telemetry-generator/**",
     ".claude/**"
   ],
   "ignoreBinaries": ["make", "migrate"],

packages/api/.env.development

@@ -20,9 +20,14 @@ REDIS_URL=redis://localhost:6379
 USAGE_STATS_ENABLED=false
 NODE_OPTIONS="--max-http-header-size=131072"
 ENABLE_SWAGGER=true
-DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""}]
-DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"}]
+DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:${HDX_DEV_CH_HTTP_PORT}","username":"default","password":""},{"name":"Local Prometheus","host":"http://localhost:${HDX_DEV_PROMETHEUS_PORT}","username":"","password":"","isPrometheusEndpoint":true}]
+DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","metadataMaterializedViews":{"keyRollupTable":"otel_logs_key_rollup_15m","kvRollupTable":"otel_logs_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","metadataMaterializedViews":{"keyRollupTable":"otel_traces_key_rollup_15m","kvRollupTable":"otel_traces_kv_rollup_15m","granularity":"15 minute"},"connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","exponential histogram":"otel_metrics_exponential_histogram","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"Timestamp","name":"JSON Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"JSON Traces","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"JSON Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"JSON Logs","metricSourceId":"JSON Metrics"},{"from":{"databaseName":"otel_json","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"JSON Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","exponential histogram":"otel_metrics_exponential_histogram"},"connection":"Local ClickHouse","logSourceId":"JSON Logs","traceSourceId":"JSON Traces"},{"from":{"databaseName":"prometheus","tableName":"prometheus"},"kind":"promql","timestampValueExpression":"timestamp","name":"Prometheus","connection":"Local Prometheus","traceSourceId":"Traces"}]
 INGESTION_API_KEY="super-secure-ingestion-api-key"
 HYPERDX_API_KEY=$INGESTION_API_KEY
 ANTHROPIC_API_KEY="your-anthropic-api-key-here"
 ENABLE_PROMQL=true
+ENABLE_SPAN_METRICS=true
+ENABLE_SPAN_METRICS_PROM_RW=true
+# Resolved API-side and inlined into the generated collector config, so this is
+# the collector's view of the network: container DNS, not a host port.
+SPAN_METRICS_PROM_RW_ENDPOINT=http://prometheus:9090/api/v1/write

packages/api/openapi.json

@@ -1642,6 +1642,16 @@
             "type": "boolean",
             "description": "Overlay the equivalent previous time period for comparison.",
             "default": false
+          },
+          "enableExemplars": {
+            "type": "boolean",
+            "description": "Overlay exemplars: markers for individual trace-linked data points. Only renders when the tile is exemplar-eligible — a single non-ratio histogram metric series with no groupBy — and is accepted but inert otherwise.\n",
+            "default": false
+          },
+          "exemplarTraceSourceId": {
+            "type": "string",
+            "description": "ID of the Trace source an exemplar marker links to. Defaults to the chart source's linked trace source when omitted.\n",
+            "example": "65f5e4a3b9e77c001a222222"
           }
         }
       },
@@ -1700,6 +1710,16 @@
           "numberFormat": {
             "$ref": "#/components/schemas/NumberFormat",
             "description": "Number formatting options for displayed values."
+          },
+          "enableExemplars": {
+            "type": "boolean",
+            "description": "Overlay exemplars: markers for individual trace-linked data points. Only renders when the tile is exemplar-eligible — a single non-ratio histogram metric series with no groupBy — and is accepted but inert otherwise.\n",
+            "default": false
+          },
+          "exemplarTraceSourceId": {
+            "type": "string",
+            "description": "ID of the Trace source an exemplar marker links to. Defaults to the chart source's linked trace source when omitted.\n",
+            "example": "65f5e4a3b9e77c001a222222"
           }
         }
       },

packages/api/src/config.ts

@@ -57,6 +57,21 @@ export const DEFAULT_SOURCES = env.DEFAULT_SOURCES;
 
 export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true';
 
+// Opt-in: have the collector derive request metrics (with trace exemplars) from
+// spans via the spanmetrics connector. Off by default; enabled in dev so the
+// telemetry-generator's traces produce coherent metric exemplars end-to-end.
+export const IS_SPAN_METRICS_ENABLED = env.ENABLE_SPAN_METRICS === 'true';
+
+// Opt-in: also remote-write the span-derived metrics (with exemplars) to a
+// Prometheus endpoint so the native Prometheus query_exemplars path can be
+// tested against real data. The endpoint is resolved here (API side) and
+// inlined into the generated collector config, so the collector container does
+// not need SPAN_METRICS_PROM_RW_ENDPOINT in its own environment. Requires the
+// endpoint to be set; without it the feature stays disabled.
+export const SPAN_METRICS_PROM_RW_ENDPOINT = env.SPAN_METRICS_PROM_RW_ENDPOINT;
+export const IS_SPAN_METRICS_PROM_RW_ENABLED =
+  env.ENABLE_SPAN_METRICS_PROM_RW === 'true' && !!SPAN_METRICS_PROM_RW_ENDPOINT;
+
 // FOR CI ONLY
 export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string;
 export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string;