From a1fc6a6f9da3b1ad36c17e2b1cea69e9f50a4f56 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Wed, 9 Aug 2023 15:22:25 +0200 Subject: [PATCH] Work around high cardinality metrics from otelcol receivers Fixes #4764 --- component/otelcol/internal/views/views.go | 53 +++++++++++++++++++++++ component/otelcol/receiver/receiver.go | 6 ++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 component/otelcol/internal/views/views.go diff --git a/component/otelcol/internal/views/views.go b/component/otelcol/internal/views/views.go new file mode 100644 index 000000000000..b78e0098f9d6 --- /dev/null +++ b/component/otelcol/internal/views/views.go @@ -0,0 +1,53 @@ +package views + +import ( + semconv "go.opentelemetry.io/collector/semconv/v1.13.0" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/instrumentation" + "go.opentelemetry.io/otel/sdk/metric" +) + +var ( + grpcScope = "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + // grpcUnacceptableKeyValues is a list of high cardinality grpc attributes that should be filtered out. + grpcUnacceptableKeyValues = []attribute.KeyValue{ + attribute.String(semconv.AttributeNetSockPeerAddr, ""), + attribute.String(semconv.AttributeNetSockPeerPort, ""), + attribute.String(semconv.AttributeNetSockPeerName, ""), + } + + httpScope = "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + // httpUnacceptableKeyValues is a list of high cardinality http attributes that should be filtered out. + httpUnacceptableKeyValues = []attribute.KeyValue{ + attribute.String(semconv.AttributeNetHostName, ""), + attribute.String(semconv.AttributeNetHostPort, ""), + attribute.String(semconv.AttributeNetSockPeerPort, ""), + attribute.String(semconv.AttributeNetSockPeerAddr, ""), + attribute.String(semconv.AttributeHTTPClientIP, ""), + } +) + +func cardinalityFilter(kvs ...attribute.KeyValue) attribute.Filter { + filter := attribute.NewSet(kvs...) + return func(kv attribute.KeyValue) bool { + return !filter.HasValue(kv.Key) + } +} + +// DropHighCardinalityServerAttributes drops certain high cardinality attributes from grpc/http server metrics +// +// This is a hopefully temporary fix to this upstream issue https://github.com/open-telemetry/opentelemetry-go-contrib/issues/3765 +func DropHighCardinalityServerAttributes() []metric.View { + var views []metric.View + + views = append(views, metric.NewView( + metric.Instrument{Scope: instrumentation.Scope{Name: grpcScope}}, + metric.Stream{AttributeFilter: cardinalityFilter(grpcUnacceptableKeyValues...)})) + + views = append(views, metric.NewView( + metric.Instrument{Scope: instrumentation.Scope{Name: httpScope}}, + metric.Stream{AttributeFilter: cardinalityFilter(httpUnacceptableKeyValues...)}, + )) + + return views +} diff --git a/component/otelcol/receiver/receiver.go b/component/otelcol/receiver/receiver.go index 64b111bfc46d..3d4c55198ca9 100644 --- a/component/otelcol/receiver/receiver.go +++ b/component/otelcol/receiver/receiver.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/agent/component/otelcol/internal/fanoutconsumer" "github.com/grafana/agent/component/otelcol/internal/lazycollector" "github.com/grafana/agent/component/otelcol/internal/scheduler" + "github.com/grafana/agent/component/otelcol/internal/views" "github.com/grafana/agent/pkg/build" "github.com/grafana/agent/pkg/util/zapadapter" "github.com/prometheus/client_golang/prometheus" @@ -125,7 +126,10 @@ func (r *Receiver) Update(args component.Arguments) error { Logger: zapadapter.New(r.opts.Logger), TracerProvider: r.opts.Tracer, - MeterProvider: metric.NewMeterProvider(metric.WithReader(promExporter)), + MeterProvider: metric.NewMeterProvider( + metric.WithReader(promExporter), + metric.WithView(views.DropHighCardinalityServerAttributes()...), + ), }, BuildInfo: otelcomponent.BuildInfo{