From 327d1122d5244e89477b42e3e63f57efd4249b43 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 1 Dec 2023 00:43:47 -0800 Subject: [PATCH 1/7] feat: extract context `aggregation_temporality`, `is_monotonic`, `unit` --- docker/ingestor/core.toml | 8 ++++++-- docker/otel-collector/config.yaml | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docker/ingestor/core.toml b/docker/ingestor/core.toml index dece4ec7e..fbece988e 100644 --- a/docker/ingestor/core.toml +++ b/docker/ingestor/core.toml @@ -709,8 +709,12 @@ source = ''' if err == null && structured.event == "metric" { # TODO: do this at extract_token .hdx_token = del(structured.fields.__HDX_API_KEY) - .dt = structured.fields.metric_type - filtered_keys = ["metric_type"] + .at = to_int(del(structured.fields.metric_aggregation_temporality)) ?? 0 + .dt = del(structured.fields.metric_type) + .im = to_bool(del(structured.fields.metric_is_monotonic)) ?? null + .u = del(structured.fields.metric_unit) + + filtered_keys = [] for_each(object(structured.fields) ?? {})-> |key, value| { if is_integer(value) || is_float(value) { filtered_keys = push(filtered_keys, key) diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index 76cb97e9a..17cfadaa0 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -30,6 +30,22 @@ processors: - key: __HDX_API_KEY from_context: authorization action: upsert + # TODO: use transform to attach __HDX_API_KEY attribute to spans/metrics/logs + transform: + error_mode: ignore + metric_statements: + - context: resource + statements: + # map metrics context to resource context (so splunk_hec will capture it) + - set(attributes["metric_aggregation_temporality"], "0") + - set(attributes["metric_unit"], "") + - set(attributes["metric_is_monotonic"], false) + - context: metric + statements: + - set(resource.attributes["metric_aggregation_temporality"], + aggregation_temporality) + - set(resource.attributes["metric_unit"], unit) + - set(resource.attributes["metric_is_monotonic"], is_monotonic) batch: memory_limiter: # 80% of maximum memory up to 2G @@ -78,7 +94,7 @@ service: exporters: [logzio/traces, logging] metrics: receivers: [otlp] - processors: [attributes/attachHdxKey, memory_limiter, batch] + processors: [attributes/attachHdxKey, transform, memory_limiter, batch] exporters: [splunk_hec, logging] logs: receivers: [otlp, fluentforward] From 50584ba68f9f921345cfd523d84fcd3a0cc6ef8b Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 1 Dec 2023 00:45:33 -0800 Subject: [PATCH 2/7] feat: ingest aggregation temporality and unit --- packages/api/src/utils/logParser.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/api/src/utils/logParser.ts b/packages/api/src/utils/logParser.ts index 7364b66f4..43183f41d 100644 --- a/packages/api/src/utils/logParser.ts +++ b/packages/api/src/utils/logParser.ts @@ -67,8 +67,10 @@ export type LogStreamModel = KeyValuePairs & export type MetricModel = { _string_attributes: Record; data_type: string; + flags: number; name: string; timestamp: number; + unit: string; value: number; }; @@ -207,14 +209,17 @@ export type VectorSpan = { }; export type VectorMetric = { + at: number; // aggregation temporality authorization?: string; b: JSONBlob; // tags dt: string; // data type hdx_platform: string; hdx_token: string; + im: boolean; // is monotonic n: string; // name ts: number; // timestamp tso: number; // observed timestamp + u: string; // unit v: number; // value }; @@ -276,8 +281,10 @@ class VectorMetricParser extends ParsingInterface { return { _string_attributes: metric.b, data_type: metric.dt, + flags: metric.at, name: metric.n, timestamp: metric.ts, + unit: metric.u, value: metric.v, }; } From 8dafdb2ddb47e225b443cd34dbc867361c462d13 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 1 Dec 2023 01:50:35 -0800 Subject: [PATCH 3/7] feat: update getMetricsTags query --- packages/api/src/clickhouse/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index 592868223..72c9ec838 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -633,6 +633,8 @@ const getMetricsTagsUncached = async (teamId: string) => { SELECT format('{} - {}', name, data_type) as name, data_type, + MAX(flags) as flags, + MAX(unit) as unit, groupUniqArray(_string_attributes) AS tags FROM ?? GROUP BY name, data_type From 625d7bcb63ab5c8fc3f9bd568fd519599e7cc08c Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Fri, 1 Dec 2023 01:53:36 -0800 Subject: [PATCH 4/7] docs: add changeset --- .changeset/grumpy-paws-do.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/grumpy-paws-do.md diff --git a/.changeset/grumpy-paws-do.md b/.changeset/grumpy-paws-do.md new file mode 100644 index 000000000..b64157615 --- /dev/null +++ b/.changeset/grumpy-paws-do.md @@ -0,0 +1,7 @@ +--- +'@hyperdx/api': minor +'@hyperdx/app': minor +--- + +feat: extract and ingest more metrics context (aggregation temporality, unit and +monotonicity) From f17d7aaa67ce31e3b0ebc8e3ff3e0501fa2a45d7 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 3 Dec 2023 15:20:34 -0800 Subject: [PATCH 5/7] feat: store is_delta and is_monotonic --- packages/api/src/utils/logParser.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/api/src/utils/logParser.ts b/packages/api/src/utils/logParser.ts index 43183f41d..9041cfd05 100644 --- a/packages/api/src/utils/logParser.ts +++ b/packages/api/src/utils/logParser.ts @@ -6,6 +6,11 @@ export type JSONBlob = Record; export type KeyPath = string[]; +export enum AggregationTemporality { + Delta = 1, + Cumulative = 2, +} + export enum LogType { Log = 'log', Metric = 'metric', @@ -67,7 +72,8 @@ export type LogStreamModel = KeyValuePairs & export type MetricModel = { _string_attributes: Record; data_type: string; - flags: number; + is_delta: boolean; + is_monotonic: boolean; name: string; timestamp: number; unit: string; @@ -281,7 +287,8 @@ class VectorMetricParser extends ParsingInterface { return { _string_attributes: metric.b, data_type: metric.dt, - flags: metric.at, + is_delta: metric.at === AggregationTemporality.Delta, + is_monotonic: metric.im, name: metric.n, timestamp: metric.ts, unit: metric.u, From 2b57db9988009e0dea8e0e1e82882986960abe19 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 3 Dec 2023 15:29:42 -0800 Subject: [PATCH 6/7] feat: update getMetricTags query --- packages/api/src/clickhouse/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index 2ee3eab86..40a78d4f2 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -633,10 +633,11 @@ const getMetricsTagsUncached = async (teamId: string) => { const query = SqlString.format( ` SELECT - format('{} - {}', name, data_type) as name, - data_type, - MAX(flags) as flags, + MAX(is_delta) as is_delta, + MAX(is_monotonic) as is_monotonic, MAX(unit) as unit, + data_type, + format('{} - {}', name, data_type) as name, groupUniqArray(_string_attributes) AS tags FROM ?? GROUP BY name, data_type From e67f964a782afbf334aa2985a43124ffbf84c58a Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Sun, 3 Dec 2023 17:14:07 -0800 Subject: [PATCH 7/7] perf: use any instead of max to select consistent fields --- packages/api/src/clickhouse/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/api/src/clickhouse/index.ts b/packages/api/src/clickhouse/index.ts index 40a78d4f2..ec66d8ea7 100644 --- a/packages/api/src/clickhouse/index.ts +++ b/packages/api/src/clickhouse/index.ts @@ -633,9 +633,9 @@ const getMetricsTagsUncached = async (teamId: string) => { const query = SqlString.format( ` SELECT - MAX(is_delta) as is_delta, - MAX(is_monotonic) as is_monotonic, - MAX(unit) as unit, + any(is_delta) as is_delta, + any(is_monotonic) as is_monotonic, + any(unit) as unit, data_type, format('{} - {}', name, data_type) as name, groupUniqArray(_string_attributes) AS tags