From 5c0aec1bc9ac60144cffa55ca45217a81c4b85e7 Mon Sep 17 00:00:00 2001 From: Jared Woelfel <89057381+jwafle@users.noreply.github.com> Date: Wed, 19 Jun 2024 00:54:21 -0400 Subject: [PATCH] [exporter/clickhouse] add temporality to histogram data models (#33425) **Description:** - Addresses lack of temporality in stored data model for `Histogram` and `Exponential Histogram` metric types in Clickhouse exporter **Link to tracking Issue:** https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/33424 --- ...houseexporter-aggregation-temporality.yaml | 33 ++++++++++++++++++ .../example/default_ddl/histogram_metrics.sql | 1 + .../example/default_ddl/sum_metrics.sql | 2 +- .../clickhouseexporter/integration_test.go | 34 ++++++++++--------- .../internal/exponential_histogram_metrics.go | 6 ++-- .../internal/histogram_metrics.go | 5 ++- .../internal/sum_metrics.go | 4 +-- 7 files changed, 63 insertions(+), 22 deletions(-) create mode 100644 .chloggen/clickhouseexporter-aggregation-temporality.yaml diff --git a/.chloggen/clickhouseexporter-aggregation-temporality.yaml b/.chloggen/clickhouseexporter-aggregation-temporality.yaml new file mode 100644 index 0000000000000..5be68bf8fb508 --- /dev/null +++ b/.chloggen/clickhouseexporter-aggregation-temporality.yaml @@ -0,0 +1,33 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add `AggregationTemporality` column to histogram and exponential histogram tables. Rename `AggTemp` column to `AggregationTemporality` in sum table. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33424] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + It's a breaking change. users who upgrade to the latest version need to alter the Clickhouse table: + ```sql + ALTER TABLE otel_metrics_exponential_histogram ADD COLUMN AggregationTemporality Int32 CODEC(ZSTD(1)); + ALTER TABLE otel_metrics_histogram ADD COLUMN AggregationTemporality Int32 CODEC(ZSTD(1)); + ALTER TABLE otel_metrics_sum RENAME COLUMN AggTemp TO AggregationTemporality; + ``` + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/clickhouseexporter/example/default_ddl/histogram_metrics.sql b/exporter/clickhouseexporter/example/default_ddl/histogram_metrics.sql index 2bb789ea05be2..055374d6bdc9c 100644 --- a/exporter/clickhouseexporter/example/default_ddl/histogram_metrics.sql +++ b/exporter/clickhouseexporter/example/default_ddl/histogram_metrics.sql @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS otel_metrics_histogram ( Flags UInt32 CODEC(ZSTD(1)), Min Float64 CODEC(ZSTD(1)), Max Float64 CODEC(ZSTD(1)), + AggregationTemporality Int32 CODEC(ZSTD(1)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, diff --git a/exporter/clickhouseexporter/example/default_ddl/sum_metrics.sql b/exporter/clickhouseexporter/example/default_ddl/sum_metrics.sql index 8cc4019616848..c0f47f173dd75 100644 --- a/exporter/clickhouseexporter/example/default_ddl/sum_metrics.sql +++ b/exporter/clickhouseexporter/example/default_ddl/sum_metrics.sql @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS otel_metrics_sum ( SpanId String, TraceId String ) CODEC(ZSTD(1)), - AggTemp Int32 CODEC(ZSTD(1)), + AggregationTemporality Int32 CODEC(ZSTD(1)), IsMonotonic Boolean CODEC(Delta, ZSTD(1)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, diff --git a/exporter/clickhouseexporter/integration_test.go b/exporter/clickhouseexporter/integration_test.go index c36f3d1098068..258465395c713 100644 --- a/exporter/clickhouseexporter/integration_test.go +++ b/exporter/clickhouseexporter/integration_test.go @@ -328,7 +328,7 @@ func verifySumMetric(t *testing.T, db *sqlx.DB) { ExemplarsValue []float64 `db:"Exemplars.Value"` ExemplarsSpanID []string `db:"Exemplars.SpanId"` ExemplarsTraceID []string `db:"Exemplars.TraceId"` - AggTemp int32 `db:"AggTemp"` + AggregationTemporality int32 `db:"AggregationTemporality"` IsMonotonic bool `db:"IsMonotonic"` } @@ -376,7 +376,7 @@ func verifySumMetric(t *testing.T, db *sqlx.DB) { } func verifyHistogramMetric(t *testing.T, db *sqlx.DB) { - type sum struct { + type histogram struct { ResourceAttributes map[string]string `db:"ResourceAttributes"` ResourceSchemaURL string `db:"ResourceSchemaUrl"` ScopeName string `db:"ScopeName"` @@ -400,14 +400,15 @@ func verifyHistogramMetric(t *testing.T, db *sqlx.DB) { ExemplarsValue []float64 `db:"Exemplars.Value"` ExemplarsSpanID []string `db:"Exemplars.SpanId"` ExemplarsTraceID []string `db:"Exemplars.TraceId"` + AggregationTemporality int32 `db:"AggregationTemporality"` Flags uint32 `db:"Flags"` Min float64 `db:"Min"` Max float64 `db:"Max"` } - var actualSum sum + var actualHistogram histogram - expectSum := sum{ + expectHistogram := histogram{ ResourceAttributes: map[string]string{ "service.name": "demo 1", "Resource Attributes 1": "value1", @@ -449,13 +450,13 @@ func verifyHistogramMetric(t *testing.T, db *sqlx.DB) { ExemplarsValue: []float64{55.22}, } - err := db.Get(&actualSum, "select * from default.otel_metrics_histogram") + err := db.Get(&actualHistogram, "select * from default.otel_metrics_histogram") require.NoError(t, err) - require.Equal(t, expectSum, actualSum) + require.Equal(t, expectHistogram, actualHistogram) } func verifyExphistogramMetric(t *testing.T, db *sqlx.DB) { - type sum struct { + type expHistogram struct { ResourceAttributes map[string]string `db:"ResourceAttributes"` ResourceSchemaURL string `db:"ResourceSchemaUrl"` ScopeName string `db:"ScopeName"` @@ -483,14 +484,15 @@ func verifyExphistogramMetric(t *testing.T, db *sqlx.DB) { ExemplarsValue []float64 `db:"Exemplars.Value"` ExemplarsSpanID []string `db:"Exemplars.SpanId"` ExemplarsTraceID []string `db:"Exemplars.TraceId"` + AggregationTemporality int32 `db:"AggregationTemporality"` Flags uint32 `db:"Flags"` Min float64 `db:"Min"` Max float64 `db:"Max"` } - var actualSum sum + var actualExpHistogram expHistogram - expectSum := sum{ + expectExpHistogram := expHistogram{ ResourceAttributes: map[string]string{ "service.name": "demo 1", "Resource Attributes 1": "value1", @@ -536,13 +538,13 @@ func verifyExphistogramMetric(t *testing.T, db *sqlx.DB) { ExemplarsValue: []float64{54}, } - err := db.Get(&actualSum, "select * from default.otel_metrics_exponential_histogram") + err := db.Get(&actualExpHistogram, "select * from default.otel_metrics_exponential_histogram") require.NoError(t, err) - require.Equal(t, expectSum, actualSum) + require.Equal(t, expectExpHistogram, actualExpHistogram) } func verifySummaryMetric(t *testing.T, db *sqlx.DB) { - type sum struct { + type summary struct { ResourceAttributes map[string]string `db:"ResourceAttributes"` ResourceSchemaURL string `db:"ResourceSchemaUrl"` ScopeName string `db:"ScopeName"` @@ -564,9 +566,9 @@ func verifySummaryMetric(t *testing.T, db *sqlx.DB) { Flags uint32 `db:"Flags"` } - var actualSum sum + var actualSummary summary - expectSum := sum{ + expectSummary := summary{ ResourceAttributes: map[string]string{ "service.name": "demo 1", "Resource Attributes 1": "value1", @@ -596,9 +598,9 @@ func verifySummaryMetric(t *testing.T, db *sqlx.DB) { Flags: 0, } - err := db.Get(&actualSum, "select * from default.otel_metrics_summary") + err := db.Get(&actualSummary, "select * from default.otel_metrics_summary") require.NoError(t, err) - require.Equal(t, expectSum, actualSum) + require.Equal(t, expectSummary, actualSummary) } func randPort() string { diff --git a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go index f6212e4e08862..d23c37d2e2fdf 100644 --- a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go @@ -51,6 +51,7 @@ CREATE TABLE IF NOT EXISTS %s_exponential_histogram %s ( Flags UInt32 CODEC(ZSTD(1)), Min Float64 CODEC(ZSTD(1)), Max Float64 CODEC(ZSTD(1)), + AggregationTemporality Int32 CODEC(ZSTD(1)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, @@ -94,7 +95,8 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; Exemplars.TraceId, Flags, Min, - Max) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)` + Max, + AggregationTemporality) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)` ) type expHistogramModel struct { @@ -135,7 +137,6 @@ func (e *expHistogramMetrics) insert(ctx context.Context, db *sql.DB) error { for i := 0; i < model.expHistogram.DataPoints().Len(); i++ { dp := model.expHistogram.DataPoints().At(i) - attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) _, err = statement.ExecContext(ctx, model.metadata.ResAttr, @@ -168,6 +169,7 @@ func (e *expHistogramMetrics) insert(ctx context.Context, db *sql.DB) error { uint32(dp.Flags()), dp.Min(), dp.Max(), + int32(model.expHistogram.AggregationTemporality()), ) if err != nil { return fmt.Errorf("ExecContext:%w", err) diff --git a/exporter/clickhouseexporter/internal/histogram_metrics.go b/exporter/clickhouseexporter/internal/histogram_metrics.go index 905b571e5c9a0..0f75d83c93e8b 100644 --- a/exporter/clickhouseexporter/internal/histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/histogram_metrics.go @@ -47,6 +47,7 @@ CREATE TABLE IF NOT EXISTS %s_histogram %s ( Flags UInt32 CODEC(ZSTD(1)), Min Float64 CODEC(ZSTD(1)), Max Float64 CODEC(ZSTD(1)), + AggregationTemporality Int32 CODEC(ZSTD(1)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, @@ -86,7 +87,8 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; Exemplars.TraceId, Flags, Min, - Max) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)` + Max, + AggregationTemporality) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)` ) type histogramModel struct { @@ -154,6 +156,7 @@ func (h *histogramMetrics) insert(ctx context.Context, db *sql.DB) error { uint32(dp.Flags()), dp.Min(), dp.Max(), + int32(model.histogram.AggregationTemporality()), ) if err != nil { return fmt.Errorf("ExecContext:%w", err) diff --git a/exporter/clickhouseexporter/internal/sum_metrics.go b/exporter/clickhouseexporter/internal/sum_metrics.go index 3276dae3a23a6..c784dfdf73254 100644 --- a/exporter/clickhouseexporter/internal/sum_metrics.go +++ b/exporter/clickhouseexporter/internal/sum_metrics.go @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS %s_sum %s ( SpanId String, TraceId String ) CODEC(ZSTD(1)), - AggTemp Int32 CODEC(ZSTD(1)), + AggregationTemporality Int32 CODEC(ZSTD(1)), IsMonotonic Boolean CODEC(Delta, ZSTD(1)), INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, @@ -79,7 +79,7 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; Exemplars.Value, Exemplars.SpanId, Exemplars.TraceId, - AggTemp, + AggregationTemporality, IsMonotonic) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)` )