Skip to content

Commit

Permalink
[exporter/clickhouse] add temporality to histogram data models (#33425)
Browse files Browse the repository at this point in the history 
**Description:**
- Addresses lack of temporality in stored data model for `Histogram` and
`Exponential Histogram` metric types in Clickhouse exporter

**Link to tracking Issue:** #33424
  • Loading branch information
@jwafle
jwafle committed Jun 19, 2024
1 parent 351f5ec commit 5c0aec1
Show file tree
Hide file tree
Showing 7 changed files with 63 additions and 22 deletions.
33 changes: 33 additions & 0 deletions .chloggen/clickhouseexporter-aggregation-temporality.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: breaking

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: clickhouseexporter

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Add `AggregationTemporality` column to histogram and exponential histogram tables. Rename `AggTemp` column to `AggregationTemporality` in sum table.

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [33424]

# (Optional) One or more lines of additional information to render under the primary note.
# These lines will be padded with 2 spaces and then inserted directly into the document.
# Use pipe (|) for multiline entries.
subtext: |
It's a breaking change. users who upgrade to the latest version need to alter the Clickhouse table:
```sql
ALTER TABLE otel_metrics_exponential_histogram ADD COLUMN AggregationTemporality Int32 CODEC(ZSTD(1));
ALTER TABLE otel_metrics_histogram ADD COLUMN AggregationTemporality Int32 CODEC(ZSTD(1));
ALTER TABLE otel_metrics_sum RENAME COLUMN AggTemp TO AggregationTemporality;
```
# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [user]
1 change: 1 addition & 0 deletions exporter/clickhouseexporter/example/default_ddl/histogram_metrics.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS otel_metrics_histogram (
Flags UInt32 CODEC(ZSTD(1)),
Min Float64 CODEC(ZSTD(1)),
Max Float64 CODEC(ZSTD(1)),
AggregationTemporality Int32 CODEC(ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
Expand Down
2 changes: 1 addition & 1 deletion exporter/clickhouseexporter/example/default_ddl/sum_metrics.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS otel_metrics_sum (
SpanId String,
TraceId String
) CODEC(ZSTD(1)),
AggTemp Int32 CODEC(ZSTD(1)),
AggregationTemporality Int32 CODEC(ZSTD(1)),
IsMonotonic Boolean CODEC(Delta, ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
Expand Down
34 changes: 18 additions & 16 deletions exporter/clickhouseexporter/integration_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,7 @@ func verifySumMetric(t *testing.T, db *sqlx.DB) {
ExemplarsValue []float64 `db:"Exemplars.Value"`
ExemplarsSpanID []string `db:"Exemplars.SpanId"`
ExemplarsTraceID []string `db:"Exemplars.TraceId"`
AggTemp int32 `db:"AggTemp"`
AggregationTemporality int32 `db:"AggregationTemporality"`
IsMonotonic bool `db:"IsMonotonic"`
}

Expand Down Expand Up @@ -376,7 +376,7 @@ func verifySumMetric(t *testing.T, db *sqlx.DB) {
}

func verifyHistogramMetric(t *testing.T, db *sqlx.DB) {
type sum struct {
type histogram struct {
ResourceAttributes map[string]string `db:"ResourceAttributes"`
ResourceSchemaURL string `db:"ResourceSchemaUrl"`
ScopeName string `db:"ScopeName"`
Expand All @@ -400,14 +400,15 @@ func verifyHistogramMetric(t *testing.T, db *sqlx.DB) {
ExemplarsValue []float64 `db:"Exemplars.Value"`
ExemplarsSpanID []string `db:"Exemplars.SpanId"`
ExemplarsTraceID []string `db:"Exemplars.TraceId"`
AggregationTemporality int32 `db:"AggregationTemporality"`
Flags uint32 `db:"Flags"`
Min float64 `db:"Min"`
Max float64 `db:"Max"`
}

var actualSum sum
var actualHistogram histogram

expectSum := sum{
expectHistogram := histogram{
ResourceAttributes: map[string]string{
"service.name": "demo 1",
"Resource Attributes 1": "value1",
Expand Down Expand Up @@ -449,13 +450,13 @@ func verifyHistogramMetric(t *testing.T, db *sqlx.DB) {
ExemplarsValue: []float64{55.22},
}

err := db.Get(&actualSum, "select * from default.otel_metrics_histogram")
err := db.Get(&actualHistogram, "select * from default.otel_metrics_histogram")
require.NoError(t, err)
require.Equal(t, expectSum, actualSum)
require.Equal(t, expectHistogram, actualHistogram)
}

func verifyExphistogramMetric(t *testing.T, db *sqlx.DB) {
type sum struct {
type expHistogram struct {
ResourceAttributes map[string]string `db:"ResourceAttributes"`
ResourceSchemaURL string `db:"ResourceSchemaUrl"`
ScopeName string `db:"ScopeName"`
Expand Down Expand Up @@ -483,14 +484,15 @@ func verifyExphistogramMetric(t *testing.T, db *sqlx.DB) {
ExemplarsValue []float64 `db:"Exemplars.Value"`
ExemplarsSpanID []string `db:"Exemplars.SpanId"`
ExemplarsTraceID []string `db:"Exemplars.TraceId"`
AggregationTemporality int32 `db:"AggregationTemporality"`
Flags uint32 `db:"Flags"`
Min float64 `db:"Min"`
Max float64 `db:"Max"`
}

var actualSum sum
var actualExpHistogram expHistogram

expectSum := sum{
expectExpHistogram := expHistogram{
ResourceAttributes: map[string]string{
"service.name": "demo 1",
"Resource Attributes 1": "value1",
Expand Down Expand Up @@ -536,13 +538,13 @@ func verifyExphistogramMetric(t *testing.T, db *sqlx.DB) {
ExemplarsValue: []float64{54},
}

err := db.Get(&actualSum, "select * from default.otel_metrics_exponential_histogram")
err := db.Get(&actualExpHistogram, "select * from default.otel_metrics_exponential_histogram")
require.NoError(t, err)
require.Equal(t, expectSum, actualSum)
require.Equal(t, expectExpHistogram, actualExpHistogram)
}

func verifySummaryMetric(t *testing.T, db *sqlx.DB) {
type sum struct {
type summary struct {
ResourceAttributes map[string]string `db:"ResourceAttributes"`
ResourceSchemaURL string `db:"ResourceSchemaUrl"`
ScopeName string `db:"ScopeName"`
Expand All @@ -564,9 +566,9 @@ func verifySummaryMetric(t *testing.T, db *sqlx.DB) {
Flags uint32 `db:"Flags"`
}

var actualSum sum
var actualSummary summary

expectSum := sum{
expectSummary := summary{
ResourceAttributes: map[string]string{
"service.name": "demo 1",
"Resource Attributes 1": "value1",
Expand Down Expand Up @@ -596,9 +598,9 @@ func verifySummaryMetric(t *testing.T, db *sqlx.DB) {
Flags: 0,
}

err := db.Get(&actualSum, "select * from default.otel_metrics_summary")
err := db.Get(&actualSummary, "select * from default.otel_metrics_summary")
require.NoError(t, err)
require.Equal(t, expectSum, actualSum)
require.Equal(t, expectSummary, actualSummary)
}

func randPort() string {
Expand Down
6 changes: 4 additions & 2 deletions exporter/clickhouseexporter/internal/exponential_histogram_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ CREATE TABLE IF NOT EXISTS %s_exponential_histogram %s (
Flags UInt32 CODEC(ZSTD(1)),
Min Float64 CODEC(ZSTD(1)),
Max Float64 CODEC(ZSTD(1)),
AggregationTemporality Int32 CODEC(ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
Expand Down Expand Up @@ -94,7 +95,8 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
Exemplars.TraceId,
Flags,
Min,
Max) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`
Max,
AggregationTemporality) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`
)

type expHistogramModel struct {
Expand Down Expand Up @@ -135,7 +137,6 @@ func (e *expHistogramMetrics) insert(ctx context.Context, db *sql.DB) error {

for i := 0; i < model.expHistogram.DataPoints().Len(); i++ {
dp := model.expHistogram.DataPoints().At(i)

attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars())
_, err = statement.ExecContext(ctx,
model.metadata.ResAttr,
Expand Down Expand Up @@ -168,6 +169,7 @@ func (e *expHistogramMetrics) insert(ctx context.Context, db *sql.DB) error {
uint32(dp.Flags()),
dp.Min(),
dp.Max(),
int32(model.expHistogram.AggregationTemporality()),
)
if err != nil {
return fmt.Errorf("ExecContext:%w", err)
Expand Down
5 changes: 4 additions & 1 deletion exporter/clickhouseexporter/internal/histogram_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ CREATE TABLE IF NOT EXISTS %s_histogram %s (
Flags UInt32 CODEC(ZSTD(1)),
Min Float64 CODEC(ZSTD(1)),
Max Float64 CODEC(ZSTD(1)),
AggregationTemporality Int32 CODEC(ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
Expand Down Expand Up @@ -86,7 +87,8 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
Exemplars.TraceId,
Flags,
Min,
Max) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`
Max,
AggregationTemporality) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`
)

type histogramModel struct {
Expand Down Expand Up @@ -154,6 +156,7 @@ func (h *histogramMetrics) insert(ctx context.Context, db *sql.DB) error {
uint32(dp.Flags()),
dp.Min(),
dp.Max(),
int32(model.histogram.AggregationTemporality()),
)
if err != nil {
return fmt.Errorf("ExecContext:%w", err)
Expand Down
4 changes: 2 additions & 2 deletions exporter/clickhouseexporter/internal/sum_metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS %s_sum %s (
SpanId String,
TraceId String
) CODEC(ZSTD(1)),
AggTemp Int32 CODEC(ZSTD(1)),
AggregationTemporality Int32 CODEC(ZSTD(1)),
IsMonotonic Boolean CODEC(Delta, ZSTD(1)),
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
Expand Down Expand Up @@ -79,7 +79,7 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
Exemplars.Value,
Exemplars.SpanId,
Exemplars.TraceId,
AggTemp,
AggregationTemporality,
IsMonotonic) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`
)

Expand Down

0 comments on commit 5c0aec1

Please sign in to comment.