From d3f88b683c26aceb5a80efcf0d83b907b3d90ff0 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Wed, 21 May 2025 22:13:24 -0700 Subject: [PATCH 1/9] feat: add init script --- docker-compose.dev.yml | 1 + docker/clickhouse/local/init-db.sh | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100755 docker/clickhouse/local/init-db.sh diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e77431560..da956300e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -56,6 +56,7 @@ services: volumes: - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml + - ./docker/clickhouse/local/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh - .volumes/ch_data_dev:/var/lib/clickhouse - .volumes/ch_logs_dev:/var/log/clickhouse-server restart: on-failure diff --git a/docker/clickhouse/local/init-db.sh b/docker/clickhouse/local/init-db.sh new file mode 100755 index 000000000..c58997856 --- /dev/null +++ b/docker/clickhouse/local/init-db.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +clickhouse client -n <<-EOSQL + CREATE DATABASE docker; + CREATE TABLE docker.docker (x Int32) ENGINE = Log; +EOSQL From ef9b18b266bd83d8c3be1772e74f0b2e60bdccb1 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 22 May 2025 00:41:04 -0700 Subject: [PATCH 2/9] feat: init otel_logs + otel_traces tables --- docker/clickhouse/local/init-db.sh | 78 ++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 3 deletions(-) diff --git a/docker/clickhouse/local/init-db.sh b/docker/clickhouse/local/init-db.sh index c58997856..83043412a 100755 --- a/docker/clickhouse/local/init-db.sh +++ b/docker/clickhouse/local/init-db.sh @@ -1,7 +1,79 @@ #!/bin/bash set -e -clickhouse client -n <<-EOSQL - CREATE DATABASE docker; - CREATE TABLE docker.docker (x Int32) ENGINE = Log; +clickhouse client -n <<'EOSQL' +CREATE TABLE default.otel_logs +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TimestampTime` DateTime DEFAULT toDateTime(Timestamp), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `TraceFlags` UInt8, + `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), + `SeverityNumber` UInt8, + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `Body` String CODEC(ZSTD(1)), + `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), + `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `__hdx_materialized_k8s.pod.name` String MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.pod.uid` String MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.node.name` String MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.namespace.name` String MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, TimestampTime) +ORDER BY (ServiceName, TimestampTime, Timestamp) +TTL TimestampTime + toIntervalDay(3) +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +CREATE TABLE default.otel_traces +( + `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + `TraceId` String CODEC(ZSTD(1)), + `SpanId` String CODEC(ZSTD(1)), + `ParentSpanId` String CODEC(ZSTD(1)), + `TraceState` String CODEC(ZSTD(1)), + `SpanName` LowCardinality(String) CODEC(ZSTD(1)), + `SpanKind` LowCardinality(String) CODEC(ZSTD(1)), + `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), + `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `ScopeName` String CODEC(ZSTD(1)), + `ScopeVersion` String CODEC(ZSTD(1)), + `SpanAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `Duration` UInt64 CODEC(ZSTD(1)), + `StatusCode` LowCardinality(String) CODEC(ZSTD(1)), + `StatusMessage` String CODEC(ZSTD(1)), + `Events.Timestamp` Array(DateTime64(9)) CODEC(ZSTD(1)), + `Events.Name` Array(LowCardinality(String)) CODEC(ZSTD(1)), + `Events.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), + `Links.TraceId` Array(String) CODEC(ZSTD(1)), + `Links.SpanId` Array(String) CODEC(ZSTD(1)), + `Links.TraceState` Array(String) CODEC(ZSTD(1)), + `Links.Attributes` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + toIntervalDay(3) +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; EOSQL From bfb1b8c23d4b31d07e87d61179ba7e334791f76c Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 22 May 2025 00:48:07 -0700 Subject: [PATCH 3/9] feat: add more k8s attribute mat fields --- docker/clickhouse/local/init-db.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/clickhouse/local/init-db.sh b/docker/clickhouse/local/init-db.sh index 83043412a..524128bf9 100755 --- a/docker/clickhouse/local/init-db.sh +++ b/docker/clickhouse/local/init-db.sh @@ -20,10 +20,13 @@ CREATE TABLE default.otel_logs `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + `__hdx_materialized_k8s.cluster.name` String MATERIALIZED ResourceAttributes['k8s.cluster.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.container.name` String MATERIALIZED ResourceAttributes['k8s.container.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.deployment.name` String MATERIALIZED ResourceAttributes['k8s.deployment.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.namespace.name` String MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), + `__hdx_materialized_k8s.node.name` String MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.name` String MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), `__hdx_materialized_k8s.pod.uid` String MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.node.name` String MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), - `__hdx_materialized_k8s.namespace.name` String MATERIALIZED ResourceAttributes['k8s.namespace.name'] CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, From f470f492712681dd4a6993011ec14e5e4ee8967a Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 22 May 2025 22:35:25 -0700 Subject: [PATCH 4/9] ci: use init-db.sh script --- docker-compose.ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 9365e9843..03e640eaf 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -29,6 +29,7 @@ services: volumes: - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml + - ./docker/clickhouse/local/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh restart: on-failure ports: - 8123:8123 # http api From 33f24422a261083f4011887932047e9fb6d7990d Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Wed, 4 Jun 2025 17:13:58 -0700 Subject: [PATCH 5/9] feat: create custom db + address some feedbacks --- docker-compose.dev.yml | 1 + docker/clickhouse/local/init-db.sh | 109 +++++++++++++++-------------- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index da956300e..2dc19f6f1 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -53,6 +53,7 @@ services: environment: # default settings CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1 + HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE: ${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} volumes: - ./docker/clickhouse/local/config.xml:/etc/clickhouse-server/config.xml - ./docker/clickhouse/local/users.xml:/etc/clickhouse-server/users.xml diff --git a/docker/clickhouse/local/init-db.sh b/docker/clickhouse/local/init-db.sh index 524128bf9..c572da8f3 100755 --- a/docker/clickhouse/local/init-db.sh +++ b/docker/clickhouse/local/init-db.sh @@ -1,32 +1,36 @@ #!/bin/bash set -e -clickhouse client -n <<'EOSQL' -CREATE TABLE default.otel_logs +DATABASE=${HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE:-default} + +clickhouse client -n < Date: Wed, 4 Jun 2025 17:37:19 -0700 Subject: [PATCH 6/9] feat: add hyperdx_sessions table + related mat fields --- docker/clickhouse/local/init-db.sh | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docker/clickhouse/local/init-db.sh b/docker/clickhouse/local/init-db.sh index c572da8f3..f2ce4eb38 100755 --- a/docker/clickhouse/local/init-db.sh +++ b/docker/clickhouse/local/init-db.sh @@ -71,7 +71,9 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_traces \`Links.SpanId\` Array(String) CODEC(ZSTD(1)), \`Links.TraceState\` Array(String) CODEC(ZSTD(1)), \`Links.Attributes\` Array(Map(LowCardinality(String), String)) CODEC(ZSTD(1)), + \`__hdx_materialized_rum.sessionId\` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, @@ -84,4 +86,41 @@ PARTITION BY toDate(Timestamp) ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) TTL toDate(Timestamp) + toIntervalDay(30) SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1; + +CREATE TABLE ${DATABASE}.hyperdx_sessions +( + \`Timestamp\` DateTime64(9) CODEC(Delta(8), ZSTD(1)), + \`TimestampTime\` DateTime DEFAULT toDateTime(Timestamp), + \`TraceId\` String CODEC(ZSTD(1)), + \`SpanId\` String CODEC(ZSTD(1)), + \`TraceFlags\` UInt8, + \`SeverityText\` LowCardinality(String) CODEC(ZSTD(1)), + \`SeverityNumber\` UInt8, + \`ServiceName\` LowCardinality(String) CODEC(ZSTD(1)), + \`Body\` String CODEC(ZSTD(1)), + \`ResourceSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)), + \`ResourceAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + \`ScopeSchemaUrl\` LowCardinality(String) CODEC(ZSTD(1)), + \`ScopeName\` String CODEC(ZSTD(1)), + \`ScopeVersion\` LowCardinality(String) CODEC(ZSTD(1)), + \`ScopeAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + \`LogAttributes\` Map(LowCardinality(String), String) CODEC(ZSTD(1)), + \`__hdx_materialized_rum.sessionId\` String MATERIALIZED ResourceAttributes['rum.sessionId'] CODEC(ZSTD(1)), + \`__hdx_materialized_type\` LowCardinality(String) MATERIALIZED toString(simpleJSONExtractInt(Body, 'type')) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_rum_session_id __hdx_materialized_rum.sessionId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, TimestampTime) +ORDER BY (ServiceName, TimestampTime, Timestamp) +TTL TimestampTime + toIntervalDay(30) +SETTINGS index_granularity = 8192, ttl_only_drop_parts = 1 EOFSQL From f1b8c42666d72bf2eaffafc1fea5d73ce93fcb64 Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 5 Jun 2025 10:25:35 -0700 Subject: [PATCH 7/9] feedback: add 'deployment.environment.name' --- docker/clickhouse/local/init-db.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/clickhouse/local/init-db.sh b/docker/clickhouse/local/init-db.sh index f2ce4eb38..17995751b 100755 --- a/docker/clickhouse/local/init-db.sh +++ b/docker/clickhouse/local/init-db.sh @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs \`__hdx_materialized_k8s.node.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.node.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.pod.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)), \`__hdx_materialized_k8s.pod.uid\` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)), + \`__hdx_materialized_deployment.environment.name\` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, From 73ef60b5481998ac0732625f3a1d7ab81ec1befa Mon Sep 17 00:00:00 2001 From: Warren <5959690+wrn14897@users.noreply.github.com> Date: Thu, 5 Jun 2025 10:28:22 -0700 Subject: [PATCH 8/9] feedback: bump default ttl to 30d --- docker/otel-collector/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/otel-collector/config.yaml b/docker/otel-collector/config.yaml index dfaa9ee66..2a0acf280 100644 --- a/docker/otel-collector/config.yaml +++ b/docker/otel-collector/config.yaml @@ -116,7 +116,7 @@ exporters: endpoint: ${env:CLICKHOUSE_ENDPOINT} password: ${env:CLICKHOUSE_PASSWORD} username: ${env:CLICKHOUSE_USER} - ttl: 72h + ttl: 720h logs_table_name: hyperdx_sessions timeout: 5s retry_on_failure: @@ -129,7 +129,7 @@ exporters: endpoint: ${env:CLICKHOUSE_ENDPOINT} password: ${env:CLICKHOUSE_PASSWORD} username: ${env:CLICKHOUSE_USER} - ttl: 72h + ttl: 720h timeout: 5s retry_on_failure: enabled: true From 56825822837193eb1600c5c9cbbfc88dde8ac17f Mon Sep 17 00:00:00 2001 From: Warren Lee <5959690+wrn14897@users.noreply.github.com> Date: Thu, 5 Jun 2025 12:28:30 -0700 Subject: [PATCH 9/9] docs: add changeset --- .changeset/quiet-singers-fry.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/quiet-singers-fry.md diff --git a/.changeset/quiet-singers-fry.md b/.changeset/quiet-singers-fry.md new file mode 100644 index 000000000..b5cda020b --- /dev/null +++ b/.changeset/quiet-singers-fry.md @@ -0,0 +1,7 @@ +--- +"@hyperdx/api": patch +"@hyperdx/app": patch +"@hyperdx/common-utils": patch +--- + +feat: introduce clickhouse db init script