From a1fc6a6f9da3b1ad36c17e2b1cea69e9f50a4f56 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Wed, 9 Aug 2023 15:22:25 +0200 Subject: [PATCH 01/24] Work around high cardinality metrics from otelcol receivers Fixes #4764 --- component/otelcol/internal/views/views.go | 53 +++++++++++++++++++++++ component/otelcol/receiver/receiver.go | 6 ++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 component/otelcol/internal/views/views.go diff --git a/component/otelcol/internal/views/views.go b/component/otelcol/internal/views/views.go new file mode 100644 index 000000000000..b78e0098f9d6 --- /dev/null +++ b/component/otelcol/internal/views/views.go @@ -0,0 +1,53 @@ +package views + +import ( + semconv "go.opentelemetry.io/collector/semconv/v1.13.0" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/instrumentation" + "go.opentelemetry.io/otel/sdk/metric" +) + +var ( + grpcScope = "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + // grpcUnacceptableKeyValues is a list of high cardinality grpc attributes that should be filtered out. + grpcUnacceptableKeyValues = []attribute.KeyValue{ + attribute.String(semconv.AttributeNetSockPeerAddr, ""), + attribute.String(semconv.AttributeNetSockPeerPort, ""), + attribute.String(semconv.AttributeNetSockPeerName, ""), + } + + httpScope = "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + // httpUnacceptableKeyValues is a list of high cardinality http attributes that should be filtered out. + httpUnacceptableKeyValues = []attribute.KeyValue{ + attribute.String(semconv.AttributeNetHostName, ""), + attribute.String(semconv.AttributeNetHostPort, ""), + attribute.String(semconv.AttributeNetSockPeerPort, ""), + attribute.String(semconv.AttributeNetSockPeerAddr, ""), + attribute.String(semconv.AttributeHTTPClientIP, ""), + } +) + +func cardinalityFilter(kvs ...attribute.KeyValue) attribute.Filter { + filter := attribute.NewSet(kvs...) + return func(kv attribute.KeyValue) bool { + return !filter.HasValue(kv.Key) + } +} + +// DropHighCardinalityServerAttributes drops certain high cardinality attributes from grpc/http server metrics +// +// This is a hopefully temporary fix to this upstream issue https://github.com/open-telemetry/opentelemetry-go-contrib/issues/3765 +func DropHighCardinalityServerAttributes() []metric.View { + var views []metric.View + + views = append(views, metric.NewView( + metric.Instrument{Scope: instrumentation.Scope{Name: grpcScope}}, + metric.Stream{AttributeFilter: cardinalityFilter(grpcUnacceptableKeyValues...)})) + + views = append(views, metric.NewView( + metric.Instrument{Scope: instrumentation.Scope{Name: httpScope}}, + metric.Stream{AttributeFilter: cardinalityFilter(httpUnacceptableKeyValues...)}, + )) + + return views +} diff --git a/component/otelcol/receiver/receiver.go b/component/otelcol/receiver/receiver.go index 64b111bfc46d..3d4c55198ca9 100644 --- a/component/otelcol/receiver/receiver.go +++ b/component/otelcol/receiver/receiver.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/agent/component/otelcol/internal/fanoutconsumer" "github.com/grafana/agent/component/otelcol/internal/lazycollector" "github.com/grafana/agent/component/otelcol/internal/scheduler" + "github.com/grafana/agent/component/otelcol/internal/views" "github.com/grafana/agent/pkg/build" "github.com/grafana/agent/pkg/util/zapadapter" "github.com/prometheus/client_golang/prometheus" @@ -125,7 +126,10 @@ func (r *Receiver) Update(args component.Arguments) error { Logger: zapadapter.New(r.opts.Logger), TracerProvider: r.opts.Tracer, - MeterProvider: metric.NewMeterProvider(metric.WithReader(promExporter)), + MeterProvider: metric.NewMeterProvider( + metric.WithReader(promExporter), + metric.WithView(views.DropHighCardinalityServerAttributes()...), + ), }, BuildInfo: otelcomponent.BuildInfo{ From 5db21619451763e4c3b449e5bb34b1af164e6330 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Wed, 9 Aug 2023 15:41:01 +0200 Subject: [PATCH 02/24] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28f4979177fa..349937d6062c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,8 @@ Main (unreleased) - Fix issue on Windows where DNS short names were unresolvable. (@rfratto) +- Drop high cardinality metric labels from otelcol receiver components in flow mode. (@glindstedt) + v0.35.3 (2023-08-09) -------------------- From 2302c8926970a2ceffbec56ae3c13650c3a64c45 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Mon, 28 Aug 2023 15:48:26 +0200 Subject: [PATCH 03/24] Add `debug_metrics` configuration block to otelcol components --- component/otelcol/config_debug_metrics.go | 16 ++++++++++++++++ component/otelcol/exporter/exporter.go | 11 ++++++++++- component/otelcol/exporter/exporter_test.go | 9 ++++++--- component/otelcol/exporter/jaeger/jaeger.go | 12 +++++++++--- .../exporter/loadbalancing/loadbalancing.go | 16 ++++++++++------ component/otelcol/exporter/logging/logging.go | 12 +++++++++--- component/otelcol/exporter/otlp/otlp.go | 12 +++++++++--- component/otelcol/exporter/otlphttp/otlphttp.go | 12 +++++++++--- component/otelcol/receiver/jaeger/jaeger.go | 12 +++++++++--- component/otelcol/receiver/kafka/kafka.go | 12 +++++++++--- .../otelcol/receiver/opencensus/opencensus.go | 12 +++++++++--- component/otelcol/receiver/otlp/otlp.go | 8 ++++++++ component/otelcol/receiver/receiver.go | 13 +++++++++---- component/otelcol/receiver/receiver_test.go | 5 ++++- component/otelcol/receiver/zipkin/zipkin.go | 12 +++++++++--- 15 files changed, 135 insertions(+), 39 deletions(-) create mode 100644 component/otelcol/config_debug_metrics.go diff --git a/component/otelcol/config_debug_metrics.go b/component/otelcol/config_debug_metrics.go new file mode 100644 index 000000000000..ca8575bee6de --- /dev/null +++ b/component/otelcol/config_debug_metrics.go @@ -0,0 +1,16 @@ +package otelcol + +// DebugMetricsArguments configures internal metrics of the components +type DebugMetricsArguments struct { + DisableHighCardinalityMetrics bool `river:"disable_high_cardinality_metrics,attr,optional"` +} + +// DefaultDebugMetricsArguments holds default settings for DebugMetricsArguments. +var DefaultDebugMetricsArguments = DebugMetricsArguments{ + DisableHighCardinalityMetrics: false, +} + +// SetToDefault implements river.Defaulter. +func (args *DebugMetricsArguments) SetToDefault() { + *args = DefaultDebugMetricsArguments +} diff --git a/component/otelcol/exporter/exporter.go b/component/otelcol/exporter/exporter.go index d2421e87a580..b33fd57ee31e 100644 --- a/component/otelcol/exporter/exporter.go +++ b/component/otelcol/exporter/exporter.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/agent/component/otelcol/internal/lazycollector" "github.com/grafana/agent/component/otelcol/internal/lazyconsumer" "github.com/grafana/agent/component/otelcol/internal/scheduler" + "github.com/grafana/agent/component/otelcol/internal/views" "github.com/grafana/agent/pkg/build" "github.com/grafana/agent/pkg/util/zapadapter" "github.com/prometheus/client_golang/prometheus" @@ -40,6 +41,9 @@ type Arguments interface { // Exporters returns the set of exporters that are exposed to the configured // component. Exporters() map[otelcomponent.DataType]map[otelcomponent.ID]otelcomponent.Component + + // DebugMetricsConfig returns the configuration for debug metrics + DebugMetricsConfig() *otelcol.DebugMetricsArguments } // Exporter is a Flow component shim which manages an OpenTelemetry Collector @@ -127,12 +131,17 @@ func (e *Exporter) Update(args component.Arguments) error { return err } + metricOpts := []metric.Option{metric.WithReader(promExporter)} + if eargs.DebugMetricsConfig().DisableHighCardinalityMetrics { + metricOpts = append(metricOpts, metric.WithView(views.DropHighCardinalityServerAttributes()...)) + } + settings := otelexporter.CreateSettings{ TelemetrySettings: otelcomponent.TelemetrySettings{ Logger: zapadapter.New(e.opts.Logger), TracerProvider: e.opts.Tracer, - MeterProvider: metric.NewMeterProvider(metric.WithReader(promExporter)), + MeterProvider: metric.NewMeterProvider(metricOpts...), }, BuildInfo: otelcomponent.BuildInfo{ diff --git a/component/otelcol/exporter/exporter_test.go b/component/otelcol/exporter/exporter_test.go index 7f37896b17be..0f4f5ae72253 100644 --- a/component/otelcol/exporter/exporter_test.go +++ b/component/otelcol/exporter/exporter_test.go @@ -121,8 +121,7 @@ func (te *testEnvironment) Start() { }() } -type fakeExporterArgs struct { -} +type fakeExporterArgs struct{} var _ exporter.Arguments = fakeExporterArgs{} @@ -138,6 +137,10 @@ func (fa fakeExporterArgs) Exporters() map[otelcomponent.DataType]map[otelcompon return nil } +func (fe fakeExporterArgs) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return nil +} + type fakeExporter struct { StartFunc func(ctx context.Context, host otelcomponent.Host) error ShutdownFunc func(ctx context.Context) error @@ -178,7 +181,7 @@ func (fe *fakeExporter) ConsumeTraces(ctx context.Context, td ptrace.Traces) err func createTestTraces() ptrace.Traces { // Matches format from the protobuf definition: // https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto - var bb = `{ + bb := `{ "resource_spans": [{ "scope_spans": [{ "spans": [{ diff --git a/component/otelcol/exporter/jaeger/jaeger.go b/component/otelcol/exporter/jaeger/jaeger.go index c5c34c37bb7d..0fc79249db2e 100644 --- a/component/otelcol/exporter/jaeger/jaeger.go +++ b/component/otelcol/exporter/jaeger/jaeger.go @@ -33,12 +33,13 @@ type Arguments struct { Queue otelcol.QueueArguments `river:"sending_queue,block,optional"` Retry otelcol.RetryArguments `river:"retry_on_failure,block,optional"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + Client GRPCClientArguments `river:"client,block"` } -var ( - _ exporter.Arguments = Arguments{} -) +var _ exporter.Arguments = Arguments{} // DefaultArguments holds default values for Arguments. var DefaultArguments = Arguments{ @@ -75,6 +76,11 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I return nil } +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} + // GRPCClientArguments is used to configure otelcol.exporter.jaeger with // component-specific defaults. type GRPCClientArguments otelcol.GRPCClientArguments diff --git a/component/otelcol/exporter/loadbalancing/loadbalancing.go b/component/otelcol/exporter/loadbalancing/loadbalancing.go index 276f855d42f8..b723043bd1ac 100644 --- a/component/otelcol/exporter/loadbalancing/loadbalancing.go +++ b/component/otelcol/exporter/loadbalancing/loadbalancing.go @@ -38,6 +38,9 @@ type Arguments struct { Protocol Protocol `river:"protocol,block"` Resolver ResolverSettings `river:"resolver,block"` RoutingKey string `river:"routing_key,attr,optional"` + + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` } var ( @@ -137,9 +140,7 @@ type DNSResolver struct { Timeout time.Duration `river:"timeout,attr,optional"` } -var ( - _ river.Defaulter = &DNSResolver{} -) +var _ river.Defaulter = &DNSResolver{} // DefaultDNSResolver holds default values for DNSResolver. var DefaultDNSResolver = DNSResolver{ @@ -172,6 +173,11 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I return nil } +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} + // GRPCClientArguments is the same as otelcol.GRPCClientArguments, but without an "endpoint" attribute type GRPCClientArguments struct { Compression otelcol.CompressionType `river:"compression,attr,optional"` @@ -190,9 +196,7 @@ type GRPCClientArguments struct { Auth *auth.Handler `river:"auth,attr,optional"` } -var ( - _ river.Defaulter = &GRPCClientArguments{} -) +var _ river.Defaulter = &GRPCClientArguments{} // Convert converts args into the upstream type. func (args *GRPCClientArguments) Convert() *otelconfiggrpc.GRPCClientSettings { diff --git a/component/otelcol/exporter/logging/logging.go b/component/otelcol/exporter/logging/logging.go index 1e717ad89d1b..02f102f3fa8e 100644 --- a/component/otelcol/exporter/logging/logging.go +++ b/component/otelcol/exporter/logging/logging.go @@ -29,11 +29,12 @@ type Arguments struct { Verbosity configtelemetry.Level `river:"verbosity,attr,optional"` SamplingInitial int `river:"sampling_initial,attr,optional"` SamplingThereafter int `river:"sampling_thereafter,attr,optional"` + + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` } -var ( - _ exporter.Arguments = Arguments{} -) +var _ exporter.Arguments = Arguments{} // DefaultArguments holds default values for Arguments. var DefaultArguments = Arguments{ @@ -65,3 +66,8 @@ func (args Arguments) Extensions() map[otelcomponent.ID]otelextension.Extension func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.ID]otelcomponent.Component { return nil } + +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} diff --git a/component/otelcol/exporter/otlp/otlp.go b/component/otelcol/exporter/otlp/otlp.go index 77d47c24fc92..f3a9195dc631 100644 --- a/component/otelcol/exporter/otlp/otlp.go +++ b/component/otelcol/exporter/otlp/otlp.go @@ -33,12 +33,13 @@ type Arguments struct { Queue otelcol.QueueArguments `river:"sending_queue,block,optional"` Retry otelcol.RetryArguments `river:"retry_on_failure,block,optional"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + Client GRPCClientArguments `river:"client,block"` } -var ( - _ exporter.Arguments = Arguments{} -) +var _ exporter.Arguments = Arguments{} // DefaultArguments holds default values for Arguments. var DefaultArguments = Arguments{ @@ -75,6 +76,11 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I return nil } +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} + // GRPCClientArguments is used to configure otelcol.exporter.otlp with // component-specific defaults. type GRPCClientArguments otelcol.GRPCClientArguments diff --git a/component/otelcol/exporter/otlphttp/otlphttp.go b/component/otelcol/exporter/otlphttp/otlphttp.go index bbc6d47535d5..4fa6a3ad39b5 100644 --- a/component/otelcol/exporter/otlphttp/otlphttp.go +++ b/component/otelcol/exporter/otlphttp/otlphttp.go @@ -32,6 +32,9 @@ type Arguments struct { Queue otelcol.QueueArguments `river:"sending_queue,block,optional"` Retry otelcol.RetryArguments `river:"retry_on_failure,block,optional"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + // The URLs to send metrics/logs/traces to. If omitted the exporter will // use Client.Endpoint by appending "/v1/metrics", "/v1/logs" or // "/v1/traces", respectively. If set, these settings override @@ -41,9 +44,7 @@ type Arguments struct { LogsEndpoint string `river:"logs_endpoint,attr,optional"` } -var ( - _ exporter.Arguments = Arguments{} -) +var _ exporter.Arguments = Arguments{} // DefaultArguments holds default values for Arguments. var DefaultArguments = Arguments{ @@ -79,6 +80,11 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I return nil } +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} + // Validate implements river.Validator. func (args *Arguments) Validate() error { if args.Client.Endpoint == "" && args.TracesEndpoint == "" && args.MetricsEndpoint == "" && args.LogsEndpoint == "" { diff --git a/component/otelcol/receiver/jaeger/jaeger.go b/component/otelcol/receiver/jaeger/jaeger.go index 9d483c608845..42fbf633c82a 100644 --- a/component/otelcol/receiver/jaeger/jaeger.go +++ b/component/otelcol/receiver/jaeger/jaeger.go @@ -31,13 +31,14 @@ func init() { type Arguments struct { Protocols ProtocolsArguments `river:"protocols,block"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` } -var ( - _ receiver.Arguments = Arguments{} -) +var _ receiver.Arguments = Arguments{} // Validate implements river.Validator. func (args *Arguments) Validate() error { @@ -210,3 +211,8 @@ func (args *ThriftBinary) Convert() *jaegerreceiver.ProtocolUDP { return args.ProtocolUDP.Convert() } + +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} diff --git a/component/otelcol/receiver/kafka/kafka.go b/component/otelcol/receiver/kafka/kafka.go index 313387a699ff..2c05b354fe11 100644 --- a/component/otelcol/receiver/kafka/kafka.go +++ b/component/otelcol/receiver/kafka/kafka.go @@ -41,13 +41,14 @@ type Arguments struct { AutoCommit AutoCommitArguments `river:"autocommit,block,optional"` MessageMarking MessageMarkingArguments `river:"message_marking,block,optional"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` } -var ( - _ receiver.Arguments = Arguments{} -) +var _ receiver.Arguments = Arguments{} // DefaultArguments holds default values for Arguments. var DefaultArguments = Arguments{ @@ -279,3 +280,8 @@ func (args MessageMarkingArguments) Convert() kafkareceiver.MessageMarking { OnError: args.IncludeUnsuccessful, } } + +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} diff --git a/component/otelcol/receiver/opencensus/opencensus.go b/component/otelcol/receiver/opencensus/opencensus.go index 9f3b6cafcb8f..e50e9d38fbc1 100644 --- a/component/otelcol/receiver/opencensus/opencensus.go +++ b/component/otelcol/receiver/opencensus/opencensus.go @@ -29,13 +29,14 @@ type Arguments struct { GRPC otelcol.GRPCServerArguments `river:",squash"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` } -var ( - _ receiver.Arguments = Arguments{} -) +var _ receiver.Arguments = Arguments{} // Default server settings. var DefaultArguments = Arguments{ @@ -75,3 +76,8 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I func (args Arguments) NextConsumers() *otelcol.ConsumerArguments { return args.Output } + +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} diff --git a/component/otelcol/receiver/otlp/otlp.go b/component/otelcol/receiver/otlp/otlp.go index 7c6252692c8a..a292903c04c6 100644 --- a/component/otelcol/receiver/otlp/otlp.go +++ b/component/otelcol/receiver/otlp/otlp.go @@ -28,6 +28,9 @@ type Arguments struct { GRPC *GRPCServerArguments `river:"grpc,block,optional"` HTTP *HTTPServerArguments `river:"http,block,optional"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` } @@ -93,3 +96,8 @@ func (args *GRPCServerArguments) SetToDefault() { func (args *HTTPServerArguments) SetToDefault() { *args = DefaultHTTPServerArguments } + +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} diff --git a/component/otelcol/receiver/receiver.go b/component/otelcol/receiver/receiver.go index 3d4c55198ca9..715e6cb5c873 100644 --- a/component/otelcol/receiver/receiver.go +++ b/component/otelcol/receiver/receiver.go @@ -44,6 +44,9 @@ type Arguments interface { // NextConsumers returns the set of consumers to send data to. NextConsumers() *otelcol.ConsumerArguments + + // DebugMetricsConfig returns the configuration for debug metrics + DebugMetricsConfig() *otelcol.DebugMetricsArguments } // Receiver is a Flow component shim which manages an OpenTelemetry Collector @@ -121,15 +124,17 @@ func (r *Receiver) Update(args component.Arguments) error { return err } + metricOpts := []metric.Option{metric.WithReader(promExporter)} + if rargs.DebugMetricsConfig().DisableHighCardinalityMetrics { + metricOpts = append(metricOpts, metric.WithView(views.DropHighCardinalityServerAttributes()...)) + } + settings := otelreceiver.CreateSettings{ TelemetrySettings: otelcomponent.TelemetrySettings{ Logger: zapadapter.New(r.opts.Logger), TracerProvider: r.opts.Tracer, - MeterProvider: metric.NewMeterProvider( - metric.WithReader(promExporter), - metric.WithView(views.DropHighCardinalityServerAttributes()...), - ), + MeterProvider: metric.NewMeterProvider(metricOpts...), }, BuildInfo: otelcomponent.BuildInfo{ diff --git a/component/otelcol/receiver/receiver_test.go b/component/otelcol/receiver/receiver_test.go index 1eaaaae64c2f..6e36e4572f55 100644 --- a/component/otelcol/receiver/receiver_test.go +++ b/component/otelcol/receiver/receiver_test.go @@ -82,7 +82,6 @@ func newTestEnvironment(t *testing.T, onTracesConsumer func(t otelconsumer.Trace _ otelcomponent.Config, t otelconsumer.Traces, ) (otelreceiver.Traces, error) { - onTracesConsumer(t) return nil, nil }, otelcomponent.StabilityLevelUndefined), @@ -127,3 +126,7 @@ func (fa fakeReceiverArgs) Exporters() map[otelcomponent.DataType]map[otelcompon func (fa fakeReceiverArgs) NextConsumers() *otelcol.ConsumerArguments { return fa.Output } + +func (fa fakeReceiverArgs) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return nil +} diff --git a/component/otelcol/receiver/zipkin/zipkin.go b/component/otelcol/receiver/zipkin/zipkin.go index 6c652840850e..dc75ffe44deb 100644 --- a/component/otelcol/receiver/zipkin/zipkin.go +++ b/component/otelcol/receiver/zipkin/zipkin.go @@ -28,13 +28,14 @@ type Arguments struct { HTTPServer otelcol.HTTPServerArguments `river:",squash"` + // DebugMetrics configures component internal metrics. Optional. + DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` } -var ( - _ receiver.Arguments = Arguments{} -) +var _ receiver.Arguments = Arguments{} // DefaultArguments holds default settings for otelcol.receiver.zipkin. var DefaultArguments = Arguments{ @@ -70,3 +71,8 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I func (args Arguments) NextConsumers() *otelcol.ConsumerArguments { return args.Output } + +// DebugMetricsConfig implements receiver.Arguments. +func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { + return args.DebugMetrics +} From 20a5252d7d0d68cfd09eb2a92d437bff2b86e4c7 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Mon, 28 Aug 2023 15:59:36 +0200 Subject: [PATCH 04/24] Revert "Update CHANGELOG.md" This reverts commit 5db21619451763e4c3b449e5bb34b1af164e6330. --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5dd55fa2938f..96ec6d6dee19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,8 +72,6 @@ Main (unreleased) - Fix issue on Windows where DNS short names were unresolvable. (@rfratto) -- Drop high cardinality metric labels from otelcol receiver components in flow mode. (@glindstedt) - v0.35.3 (2023-08-09) -------------------- From 53c3d735fb911dfe96f482dc3408d1c9813e5c7c Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Mon, 28 Aug 2023 16:37:49 +0200 Subject: [PATCH 05/24] fix nilpointer --- component/otelcol/exporter/exporter_test.go | 2 +- component/otelcol/receiver/receiver_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/component/otelcol/exporter/exporter_test.go b/component/otelcol/exporter/exporter_test.go index 0f4f5ae72253..3a4a1d655dfd 100644 --- a/component/otelcol/exporter/exporter_test.go +++ b/component/otelcol/exporter/exporter_test.go @@ -138,7 +138,7 @@ func (fa fakeExporterArgs) Exporters() map[otelcomponent.DataType]map[otelcompon } func (fe fakeExporterArgs) DebugMetricsConfig() *otelcol.DebugMetricsArguments { - return nil + return &otelcol.DefaultDebugMetricsArguments } type fakeExporter struct { diff --git a/component/otelcol/receiver/receiver_test.go b/component/otelcol/receiver/receiver_test.go index 6e36e4572f55..a1024cb94960 100644 --- a/component/otelcol/receiver/receiver_test.go +++ b/component/otelcol/receiver/receiver_test.go @@ -128,5 +128,5 @@ func (fa fakeReceiverArgs) NextConsumers() *otelcol.ConsumerArguments { } func (fa fakeReceiverArgs) DebugMetricsConfig() *otelcol.DebugMetricsArguments { - return nil + return &otelcol.DefaultDebugMetricsArguments } From 4f5c09e6f16c636ab8bf6585770cc8ee395a2c38 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Tue, 29 Aug 2023 10:50:11 +0200 Subject: [PATCH 06/24] refactor to not use pointers for debug_metrics block --- component/otelcol/exporter/exporter.go | 2 +- component/otelcol/exporter/exporter_test.go | 4 ++-- component/otelcol/exporter/jaeger/jaeger.go | 4 ++-- component/otelcol/exporter/loadbalancing/loadbalancing.go | 4 ++-- component/otelcol/exporter/logging/logging.go | 4 ++-- component/otelcol/exporter/otlp/otlp.go | 4 ++-- component/otelcol/exporter/otlphttp/otlphttp.go | 4 ++-- component/otelcol/receiver/jaeger/jaeger.go | 4 ++-- component/otelcol/receiver/kafka/kafka.go | 4 ++-- component/otelcol/receiver/opencensus/opencensus.go | 4 ++-- component/otelcol/receiver/otlp/otlp.go | 4 ++-- component/otelcol/receiver/receiver.go | 2 +- component/otelcol/receiver/receiver_test.go | 4 ++-- component/otelcol/receiver/zipkin/zipkin.go | 4 ++-- 14 files changed, 26 insertions(+), 26 deletions(-) diff --git a/component/otelcol/exporter/exporter.go b/component/otelcol/exporter/exporter.go index b33fd57ee31e..8333836cda16 100644 --- a/component/otelcol/exporter/exporter.go +++ b/component/otelcol/exporter/exporter.go @@ -43,7 +43,7 @@ type Arguments interface { Exporters() map[otelcomponent.DataType]map[otelcomponent.ID]otelcomponent.Component // DebugMetricsConfig returns the configuration for debug metrics - DebugMetricsConfig() *otelcol.DebugMetricsArguments + DebugMetricsConfig() otelcol.DebugMetricsArguments } // Exporter is a Flow component shim which manages an OpenTelemetry Collector diff --git a/component/otelcol/exporter/exporter_test.go b/component/otelcol/exporter/exporter_test.go index 3a4a1d655dfd..be07ab2b48f7 100644 --- a/component/otelcol/exporter/exporter_test.go +++ b/component/otelcol/exporter/exporter_test.go @@ -137,8 +137,8 @@ func (fa fakeExporterArgs) Exporters() map[otelcomponent.DataType]map[otelcompon return nil } -func (fe fakeExporterArgs) DebugMetricsConfig() *otelcol.DebugMetricsArguments { - return &otelcol.DefaultDebugMetricsArguments +func (fe fakeExporterArgs) DebugMetricsConfig() otelcol.DebugMetricsArguments { + return otelcol.DefaultDebugMetricsArguments } type fakeExporter struct { diff --git a/component/otelcol/exporter/jaeger/jaeger.go b/component/otelcol/exporter/jaeger/jaeger.go index 0fc79249db2e..dc64a2c54172 100644 --- a/component/otelcol/exporter/jaeger/jaeger.go +++ b/component/otelcol/exporter/jaeger/jaeger.go @@ -34,7 +34,7 @@ type Arguments struct { Retry otelcol.RetryArguments `river:"retry_on_failure,block,optional"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` Client GRPCClientArguments `river:"client,block"` } @@ -77,7 +77,7 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/exporter/loadbalancing/loadbalancing.go b/component/otelcol/exporter/loadbalancing/loadbalancing.go index b723043bd1ac..ed1cff889eea 100644 --- a/component/otelcol/exporter/loadbalancing/loadbalancing.go +++ b/component/otelcol/exporter/loadbalancing/loadbalancing.go @@ -40,7 +40,7 @@ type Arguments struct { RoutingKey string `river:"routing_key,attr,optional"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` } var ( @@ -174,7 +174,7 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/exporter/logging/logging.go b/component/otelcol/exporter/logging/logging.go index 02f102f3fa8e..23faa7cb69ec 100644 --- a/component/otelcol/exporter/logging/logging.go +++ b/component/otelcol/exporter/logging/logging.go @@ -31,7 +31,7 @@ type Arguments struct { SamplingThereafter int `river:"sampling_thereafter,attr,optional"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` } var _ exporter.Arguments = Arguments{} @@ -68,6 +68,6 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/exporter/otlp/otlp.go b/component/otelcol/exporter/otlp/otlp.go index f3a9195dc631..09c6aa332f8d 100644 --- a/component/otelcol/exporter/otlp/otlp.go +++ b/component/otelcol/exporter/otlp/otlp.go @@ -34,7 +34,7 @@ type Arguments struct { Retry otelcol.RetryArguments `river:"retry_on_failure,block,optional"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` Client GRPCClientArguments `river:"client,block"` } @@ -77,7 +77,7 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/exporter/otlphttp/otlphttp.go b/component/otelcol/exporter/otlphttp/otlphttp.go index 4fa6a3ad39b5..60d523394545 100644 --- a/component/otelcol/exporter/otlphttp/otlphttp.go +++ b/component/otelcol/exporter/otlphttp/otlphttp.go @@ -33,7 +33,7 @@ type Arguments struct { Retry otelcol.RetryArguments `river:"retry_on_failure,block,optional"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` // The URLs to send metrics/logs/traces to. If omitted the exporter will // use Client.Endpoint by appending "/v1/metrics", "/v1/logs" or @@ -81,7 +81,7 @@ func (args Arguments) Exporters() map[otelcomponent.DataType]map[otelcomponent.I } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/receiver/jaeger/jaeger.go b/component/otelcol/receiver/jaeger/jaeger.go index 42fbf633c82a..2cebb37b9114 100644 --- a/component/otelcol/receiver/jaeger/jaeger.go +++ b/component/otelcol/receiver/jaeger/jaeger.go @@ -32,7 +32,7 @@ type Arguments struct { Protocols ProtocolsArguments `river:"protocols,block"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` @@ -213,6 +213,6 @@ func (args *ThriftBinary) Convert() *jaegerreceiver.ProtocolUDP { } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/receiver/kafka/kafka.go b/component/otelcol/receiver/kafka/kafka.go index 2c05b354fe11..6c89e3e719bd 100644 --- a/component/otelcol/receiver/kafka/kafka.go +++ b/component/otelcol/receiver/kafka/kafka.go @@ -42,7 +42,7 @@ type Arguments struct { MessageMarking MessageMarkingArguments `river:"message_marking,block,optional"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` @@ -282,6 +282,6 @@ func (args MessageMarkingArguments) Convert() kafkareceiver.MessageMarking { } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/receiver/opencensus/opencensus.go b/component/otelcol/receiver/opencensus/opencensus.go index e50e9d38fbc1..7f4c64ee0ace 100644 --- a/component/otelcol/receiver/opencensus/opencensus.go +++ b/component/otelcol/receiver/opencensus/opencensus.go @@ -30,7 +30,7 @@ type Arguments struct { GRPC otelcol.GRPCServerArguments `river:",squash"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` @@ -78,6 +78,6 @@ func (args Arguments) NextConsumers() *otelcol.ConsumerArguments { } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/receiver/otlp/otlp.go b/component/otelcol/receiver/otlp/otlp.go index a292903c04c6..4fa33c18202d 100644 --- a/component/otelcol/receiver/otlp/otlp.go +++ b/component/otelcol/receiver/otlp/otlp.go @@ -29,7 +29,7 @@ type Arguments struct { HTTP *HTTPServerArguments `river:"http,block,optional"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` @@ -98,6 +98,6 @@ func (args *HTTPServerArguments) SetToDefault() { } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } diff --git a/component/otelcol/receiver/receiver.go b/component/otelcol/receiver/receiver.go index 715e6cb5c873..399c771c25fd 100644 --- a/component/otelcol/receiver/receiver.go +++ b/component/otelcol/receiver/receiver.go @@ -46,7 +46,7 @@ type Arguments interface { NextConsumers() *otelcol.ConsumerArguments // DebugMetricsConfig returns the configuration for debug metrics - DebugMetricsConfig() *otelcol.DebugMetricsArguments + DebugMetricsConfig() otelcol.DebugMetricsArguments } // Receiver is a Flow component shim which manages an OpenTelemetry Collector diff --git a/component/otelcol/receiver/receiver_test.go b/component/otelcol/receiver/receiver_test.go index a1024cb94960..8d4da7561ca5 100644 --- a/component/otelcol/receiver/receiver_test.go +++ b/component/otelcol/receiver/receiver_test.go @@ -127,6 +127,6 @@ func (fa fakeReceiverArgs) NextConsumers() *otelcol.ConsumerArguments { return fa.Output } -func (fa fakeReceiverArgs) DebugMetricsConfig() *otelcol.DebugMetricsArguments { - return &otelcol.DefaultDebugMetricsArguments +func (fa fakeReceiverArgs) DebugMetricsConfig() otelcol.DebugMetricsArguments { + return otelcol.DefaultDebugMetricsArguments } diff --git a/component/otelcol/receiver/zipkin/zipkin.go b/component/otelcol/receiver/zipkin/zipkin.go index dc75ffe44deb..1727d38a0d05 100644 --- a/component/otelcol/receiver/zipkin/zipkin.go +++ b/component/otelcol/receiver/zipkin/zipkin.go @@ -29,7 +29,7 @@ type Arguments struct { HTTPServer otelcol.HTTPServerArguments `river:",squash"` // DebugMetrics configures component internal metrics. Optional. - DebugMetrics *otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` + DebugMetrics otelcol.DebugMetricsArguments `river:"debug_metrics,block,optional"` // Output configures where to send received data. Required. Output *otelcol.ConsumerArguments `river:"output,block"` @@ -73,6 +73,6 @@ func (args Arguments) NextConsumers() *otelcol.ConsumerArguments { } // DebugMetricsConfig implements receiver.Arguments. -func (args Arguments) DebugMetricsConfig() *otelcol.DebugMetricsArguments { +func (args Arguments) DebugMetricsConfig() otelcol.DebugMetricsArguments { return args.DebugMetrics } From a06d0b15e67a414f9cc0902e22f18274704f22b0 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Wed, 30 Aug 2023 08:59:47 +0200 Subject: [PATCH 07/24] Update component/otelcol/internal/views/views.go Co-authored-by: Paulin Todev --- component/otelcol/internal/views/views.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/component/otelcol/internal/views/views.go b/component/otelcol/internal/views/views.go index b78e0098f9d6..7d2804ec342b 100644 --- a/component/otelcol/internal/views/views.go +++ b/component/otelcol/internal/views/views.go @@ -36,7 +36,11 @@ func cardinalityFilter(kvs ...attribute.KeyValue) attribute.Filter { // DropHighCardinalityServerAttributes drops certain high cardinality attributes from grpc/http server metrics // -// This is a hopefully temporary fix to this upstream issue https://github.com/open-telemetry/opentelemetry-go-contrib/issues/3765 +// This is a fix to an upstream issue: +// https://github.com/open-telemetry/opentelemetry-go-contrib/issues/3765 +// The long-term solution for the Collector is to set view settings in the Collector config: +// https://github.com/open-telemetry/opentelemetry-collector/issues/7517#issuecomment-1511168350 +// In the future, when Collector supports such config, we may want to support similar view settings in the Agent. func DropHighCardinalityServerAttributes() []metric.View { var views []metric.View From 97a7bd2192159adea7e3bf21d4afee726c9544d6 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Wed, 30 Aug 2023 09:22:05 +0200 Subject: [PATCH 08/24] add docs --- .../components/otelcol.exporter.jaeger.md | 6 ++++++ .../otelcol.exporter.loadbalancing.md | 6 ++++++ .../components/otelcol.exporter.logging.md | 18 +++++++++++++++++ .../components/otelcol.exporter.otlp.md | 6 ++++++ .../components/otelcol.exporter.otlphttp.md | 5 +++++ .../components/otelcol.receiver.jaeger.md | 6 ++++++ .../components/otelcol.receiver.kafka.md | 6 ++++++ .../components/otelcol.receiver.opencensus.md | 6 ++++++ .../components/otelcol.receiver.otlp.md | 6 ++++++ .../components/otelcol.receiver.zipkin.md | 6 ++++++ .../components/otelcol-debug-metrics-block.md | 20 +++++++++++++++++++ 11 files changed, 91 insertions(+) create mode 100644 docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md diff --git a/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md b/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md index 99a0d963084a..b94e9e8a3a30 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md @@ -47,6 +47,7 @@ client > tls | [tls][] | Configures TLS for the gRPC client. | no client > keepalive | [keepalive][] | Configures keepalive settings for the gRPC client. | no sending_queue | [sending_queue][] | Configures batching of data before sending. | no retry_on_failure | [retry_on_failure][] | Configures retry mechanism for failed requests. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `client > tls` refers to a `tls` block defined inside a `client` block. @@ -56,6 +57,7 @@ refers to a `tls` block defined inside a `client` block. [keepalive]: #keepalive-block [sending_queue]: #sending_queue-block [retry_on_failure]: #retry_on_failure-block +[debug_metrics]: #debug_metrics-block ### client block @@ -134,6 +136,10 @@ retried. {{< docs/shared lookup="flow/reference/components/otelcol-retry-block.md" source="agent" >}} +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ## Exported fields The following fields are exported and can be referenced by other components: diff --git a/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md b/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md index 15ca86872a69..47a3b54b702e 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md @@ -82,6 +82,7 @@ protocol > otlp > client > tls | [tls][] | Configures TLS for the gRPC client. | protocol > otlp > client > keepalive | [keepalive][] | Configures keepalive settings for the gRPC client. | no protocol > otlp > queue | [queue][] | Configures batching of data before sending. | no protocol > otlp > retry | [retry][] | Configures retry mechanism for failed requests. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `resolver > static` refers to a `static` block defined inside a `resolver` block. @@ -96,6 +97,7 @@ refers to a `static` block defined inside a `resolver` block. [keepalive]: #keepalive-block [queue]: #queue-block [retry]: #retry-block +[debug_metrics]: #debug_metrics-block ### resolver block @@ -215,6 +217,10 @@ retried. {{< docs/shared lookup="flow/reference/components/otelcol-retry-block.md" source="agent" >}} +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ## Exported fields The following fields are exported and can be referenced by other components: diff --git a/docs/sources/flow/reference/components/otelcol.exporter.logging.md b/docs/sources/flow/reference/components/otelcol.exporter.logging.md index 2f588aa6b8c8..cd806bd7e6b5 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.logging.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.logging.md @@ -38,6 +38,24 @@ Name | Type | Description | Default | Required The `verbosity` argument must be one of `"basic"`, `"normal"`, or `"detailed"`. +## Blocks + +The following blocks are supported inside the definition of +`otelcol.exporter.logging`: + +Hierarchy | Block | Description | Required +--------- | ----- | ----------- | -------- +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no + +The `>` symbol indicates deeper levels of nesting. For example, `client > tls` +refers to a `tls` block defined inside a `client` block. + +[debug_metrics]: #debug_metrics-block + +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ## Exported fields The following fields are exported and can be referenced by other components: diff --git a/docs/sources/flow/reference/components/otelcol.exporter.otlp.md b/docs/sources/flow/reference/components/otelcol.exporter.otlp.md index 1917e71623a6..91498b1bfc87 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.otlp.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.otlp.md @@ -45,6 +45,7 @@ client > tls | [tls][] | Configures TLS for the gRPC client. | no client > keepalive | [keepalive][] | Configures keepalive settings for the gRPC client. | no sending_queue | [sending_queue][] | Configures batching of data before sending. | no retry_on_failure | [retry_on_failure][] | Configures retry mechanism for failed requests. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `client > tls` refers to a `tls` block defined inside a `client` block. @@ -54,6 +55,7 @@ refers to a `tls` block defined inside a `client` block. [keepalive]: #keepalive-block [sending_queue]: #sending_queue-block [retry_on_failure]: #retry_on_failure-block +[debug_metrics]: #debug_metrics-block ### client block @@ -137,6 +139,10 @@ retried. {{< docs/shared lookup="flow/reference/components/otelcol-retry-block.md" source="agent" >}} +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ## Exported fields The following fields are exported and can be referenced by other components: diff --git a/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md b/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md index 03e4db3cc0cf..c47feebdfd36 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md @@ -50,6 +50,7 @@ client | [client][] | Configures the HTTP server to send telemetry dat client > tls | [tls][] | Configures TLS for the HTTP client. | no sending_queue | [sending_queue][] | Configures batching of data before sending. | no retry_on_failure | [retry_on_failure][] | Configures retry mechanism for failed requests. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `client > tls` refers to a `tls` block defined inside a `client` block. @@ -58,6 +59,7 @@ refers to a `tls` block defined inside a `client` block. [tls]: #tls-block [sending_queue]: #sending_queue-block [retry_on_failure]: #retry_on_failure-block +[debug_metrics]: #debug_metrics-block ### client block @@ -101,6 +103,9 @@ The `retry_on_failure` block configures how failed requests to the HTTP server a retried. {{< docs/shared lookup="flow/reference/components/otelcol-retry-block.md" source="agent" >}} +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} ## Exported fields diff --git a/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md b/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md index 7cfde39b6239..e92eff33fe43 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md @@ -57,6 +57,7 @@ protocols > thrift_http > tls | [tls][] | Configures TLS for the Thrift HTTP ser protocols > thrift_http > cors | [cors][] | Configures CORS for the Thrift HTTP server. | no protocols > thrift_binary | [thrift_binary][] | Configures a Thrift binary UDP server to receive traces. | no protocols > thrift_compact | [thrift_compact][] | Configures a Thrift compact UDP server to receive traces. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no output | [output][] | Configures where to send received telemetry data. | yes The `>` symbol indicates deeper levels of nesting. For example, `protocols > @@ -72,6 +73,7 @@ grpc` refers to a `grpc` block defined inside a `protocols` block. [cors]: #cors-block [thrift_binary]: #thrift_binary-block [thrift_compact]: #thrift_compact-block +[debug_metrics]: #debug_metrics-block [output]: #output-block ### protocols block @@ -213,6 +215,10 @@ Name | Type | Description | Default | Required `workers` | `number` | Number of workers to concurrently read from the message queue. | `10` | no `socket_buffer_size` | `string` | Buffer to allocate for the UDP socket. | | no +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ### output block {{< docs/shared lookup="flow/reference/components/output-block.md" source="agent" >}} diff --git a/docs/sources/flow/reference/components/otelcol.receiver.kafka.md b/docs/sources/flow/reference/components/otelcol.receiver.kafka.md index 5519107413cc..36cbb6d654c8 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.kafka.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.kafka.md @@ -80,6 +80,7 @@ metadata | [metadata][] | Configures how to retrieve metadata from Kafka brokers metadata > retry | [retry][] | Configures how to retry metadata retrieval. | no autocommit | [autocommit][] | Configures how to automatically commit updated topic offsets to back to the Kafka brokers. | no message_marking | [message_marking][] | Configures when Kafka messages are marked as read. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no output | [output][] | Configures where to send received telemetry data. | yes The `>` symbol indicates deeper levels of nesting. For example, @@ -96,6 +97,7 @@ The `>` symbol indicates deeper levels of nesting. For example, [retry]: #retry-block [autocommit]: #autocommit-block [message_marking]: #message_marking-block +[debug_metrics]: #debug_metrics-block [output]: #output-block ### authentication block @@ -249,6 +251,10 @@ has no effect if `after_execution` is `false`. > to `false` can block the entire Kafka partition if message processing returns > a permanent error, such as failing to decode. +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ### output block {{< docs/shared lookup="flow/reference/components/output-block.md" source="agent" >}} diff --git a/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md b/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md index 91ef8f96dd7a..bf9349e960a2 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md @@ -67,6 +67,7 @@ tls | [tls][] | Configures TLS for the gRPC server. | no keepalive | [keepalive][] | Configures keepalive settings for the configured server. | no keepalive > server_parameters | [server_parameters][] | Server parameters used to configure keepalive settings. | no keepalive > enforcement_policy | [enforcement_policy][] | Enforcement policy for keepalive settings. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no output | [output][] | Configures where to send received telemetry data. | yes The `>` symbol indicates deeper levels of nesting. For example, `grpc > tls` @@ -76,6 +77,7 @@ refers to a `tls` block defined inside a `grpc` block. [keepalive]: #keepalive-block [server_parameters]: #server_parameters-block [enforcement_policy]: #enforcement_policy-block +[debug_metrics]: #debug_metrics-block [output]: #output-block ### tls block @@ -121,6 +123,10 @@ Name | Type | Description | Default | Required `min_time` | `duration` | Minimum time clients should wait before sending a keepalive ping. | `"5m"` | no `permit_without_stream` | `boolean` | Allow clients to send keepalive pings when there are no active streams. | `false` | no +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ### output block {{< docs/shared lookup="flow/reference/components/output-block.md" source="agent" >}} diff --git a/docs/sources/flow/reference/components/otelcol.receiver.otlp.md b/docs/sources/flow/reference/components/otelcol.receiver.otlp.md index e5f93eed0aa1..22390d574f38 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.otlp.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.otlp.md @@ -50,6 +50,7 @@ grpc > keepalive > enforcement_policy | [enforcement_policy][] | Enforcement pol http | [http][] | Configures the HTTP server to receive telemetry data. | no http > tls | [tls][] | Configures TLS for the HTTP server. | no http > cors | [cors][] | Configures CORS for the HTTP server. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no output | [output][] | Configures where to send received telemetry data. | yes The `>` symbol indicates deeper levels of nesting. For example, `grpc > tls` @@ -62,6 +63,7 @@ refers to a `tls` block defined inside a `grpc` block. [enforcement_policy]: #enforcement_policy-block [http]: #http-block [cors]: #cors-block +[debug_metrics]: #debug_metrics-block [output]: #output-block ### grpc block @@ -169,6 +171,10 @@ CORS request. The following headers are always implicitly allowed: If `allowed_headers` includes `"*"`, all headers are permitted. +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ### output block {{< docs/shared lookup="flow/reference/components/output-block.md" source="agent" >}} diff --git a/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md b/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md index 9ff5dd20303c..1ffe9d07a716 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md @@ -49,6 +49,7 @@ Hierarchy | Block | Description | Required --------- | ----- | ----------- | -------- tls | [tls][] | Configures TLS for the HTTP server. | no cors | [cors][] | Configures CORS for the HTTP server. | no +debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no output | [output][] | Configures where to send received traces. | yes The `>` symbol indicates deeper levels of nesting. For example, `grpc > tls` @@ -56,6 +57,7 @@ refers to a `tls` block defined inside a `grpc` block. [tls]: #tls-block [cors]: #cors-block +[debug_metrics]: #debug_metrics-block [output]: #output-block ### tls block @@ -87,6 +89,10 @@ CORS request. The following headers are always implicitly allowed: If `allowed_headers` includes `"*"`, all headers are permitted. +### debug_metrics block + +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} + ### output block {{< docs/shared lookup="flow/reference/components/output-block.md" source="agent" >}} diff --git a/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md new file mode 100644 index 000000000000..11e8eb3aecbf --- /dev/null +++ b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md @@ -0,0 +1,20 @@ +--- +aliases: +- /docs/agent/shared/flow/reference/components/otelcol-debug-metrics-block/ +canonical: https://grafana.com/docs/agent/latest/shared/flow/reference/components/otelcol-debug-metrics-block/ +headless: true +--- + +The `debug_metrics` block configures the metrics which this component generates to monitor its state. + +The following arguments are supported: + +Name | Type | Description | Default | Required +---- | ---- | ----------- | ------- | -------- +`disable_high_cardinality_metrics` | `boolean` | Whether to disable certain high cardinality metrics. | `false` | no + +`disable_high_cardinality_metrics` is the Grafana Agent equivalent to the +`telemetry.useOtelForInternalMetrics` feature gate in the OpenTelemetry +Collector. It removes attributes which could case high cardinality metrics. For +example, attributes with IP addresses and port numbers in metrics about HTTP +and gRPC connections will be removed. From 8be5079801cbffc6eee0bde6bb12ebf7011ae18f Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Thu, 31 Aug 2023 10:52:15 +0200 Subject: [PATCH 09/24] fix lint --- component/otelcol/receiver/receiver_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/component/otelcol/receiver/receiver_test.go b/component/otelcol/receiver/receiver_test.go index 8d4da7561ca5..f1836e0edf78 100644 --- a/component/otelcol/receiver/receiver_test.go +++ b/component/otelcol/receiver/receiver_test.go @@ -82,6 +82,7 @@ func newTestEnvironment(t *testing.T, onTracesConsumer func(t otelconsumer.Trace _ otelcomponent.Config, t otelconsumer.Traces, ) (otelreceiver.Traces, error) { + onTracesConsumer(t) return nil, nil }, otelcomponent.StabilityLevelUndefined), From de21b719ee3ba1ce80024b30df3f330c0b985b36 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Thu, 31 Aug 2023 11:29:33 +0200 Subject: [PATCH 10/24] add debug_metrics into some component tests --- component/otelcol/exporter/otlp/otlp_test.go | 7 ++++++- component/otelcol/receiver/zipkin/zipkin_test.go | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/component/otelcol/exporter/otlp/otlp_test.go b/component/otelcol/exporter/otlp/otlp_test.go index 33ec79901fec..0bb24e4d4e34 100644 --- a/component/otelcol/exporter/otlp/otlp_test.go +++ b/component/otelcol/exporter/otlp/otlp_test.go @@ -45,9 +45,14 @@ func Test(t *testing.T) { insecure_skip_verify = true } } + + debug_metrics { + disable_high_cardinality_metrics = true + } `, tracesServer) var args otlp.Arguments require.NoError(t, river.Unmarshal([]byte(cfg), &args)) + require.Equal(t, args.DebugMetricsConfig().DisableHighCardinalityMetrics, true) go func() { err := ctrl.Run(ctx, args) @@ -121,7 +126,7 @@ func (ms *mockTracesReceiver) Export(_ context.Context, req ptraceotlp.ExportReq func createTestTraces() ptrace.Traces { // Matches format from the protobuf definition: // https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto - var bb = `{ + bb := `{ "resource_spans": [{ "scope_spans": [{ "spans": [{ diff --git a/component/otelcol/receiver/zipkin/zipkin_test.go b/component/otelcol/receiver/zipkin/zipkin_test.go index 79841e0f7be5..b4086a836733 100644 --- a/component/otelcol/receiver/zipkin/zipkin_test.go +++ b/component/otelcol/receiver/zipkin/zipkin_test.go @@ -51,11 +51,16 @@ func TestArguments_UnmarshalRiver(t *testing.T) { parse_string_tags = true + debug_metrics { + disable_high_cardinality_metrics = true + } + output { /* no-op */ } `, httpAddr) var args zipkin.Arguments require.NoError(t, river.Unmarshal([]byte(in), &args)) + require.Equal(t, args.DebugMetricsConfig().DisableHighCardinalityMetrics, true) ext, err := args.Convert() require.NoError(t, err) otelArgs, ok := (ext).(*zipkinreceiver.Config) From 583fbc18a8e5c305eca26bb3e4142e3b11e8f747 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Thu, 31 Aug 2023 14:14:33 +0200 Subject: [PATCH 11/24] align shared docs references with changes made in #4921 --- .../flow/reference/components/otelcol.exporter.jaeger.md | 2 +- .../flow/reference/components/otelcol.exporter.loadbalancing.md | 2 +- .../flow/reference/components/otelcol.exporter.logging.md | 2 +- docs/sources/flow/reference/components/otelcol.exporter.otlp.md | 2 +- .../flow/reference/components/otelcol.exporter.otlphttp.md | 2 +- .../flow/reference/components/otelcol.receiver.jaeger.md | 2 +- .../sources/flow/reference/components/otelcol.receiver.kafka.md | 2 +- .../flow/reference/components/otelcol.receiver.opencensus.md | 2 +- docs/sources/flow/reference/components/otelcol.receiver.otlp.md | 2 +- .../flow/reference/components/otelcol.receiver.zipkin.md | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md b/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md index 8e854ef535b5..2fffdb05f9fc 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md @@ -138,7 +138,7 @@ retried. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ## Exported fields diff --git a/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md b/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md index fd9f341af38f..024874b508ed 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md @@ -219,7 +219,7 @@ retried. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ## Exported fields diff --git a/docs/sources/flow/reference/components/otelcol.exporter.logging.md b/docs/sources/flow/reference/components/otelcol.exporter.logging.md index cd806bd7e6b5..1ed90bfd5423 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.logging.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.logging.md @@ -54,7 +54,7 @@ refers to a `tls` block defined inside a `client` block. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ## Exported fields diff --git a/docs/sources/flow/reference/components/otelcol.exporter.otlp.md b/docs/sources/flow/reference/components/otelcol.exporter.otlp.md index 9304d0ae8618..f12d6784db34 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.otlp.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.otlp.md @@ -141,7 +141,7 @@ retried. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ## Exported fields diff --git a/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md b/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md index 0f7cfc40747d..f87eeb1dbf4e 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md @@ -106,7 +106,7 @@ retried. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ## Exported fields diff --git a/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md b/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md index d66f6a64b2cd..243964b50f4f 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md @@ -217,7 +217,7 @@ Name | Type | Description | Default | Required ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ### output block diff --git a/docs/sources/flow/reference/components/otelcol.receiver.kafka.md b/docs/sources/flow/reference/components/otelcol.receiver.kafka.md index 1a242e02985c..1d9572a6d71e 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.kafka.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.kafka.md @@ -253,7 +253,7 @@ has no effect if `after_execution` is `false`. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ### output block diff --git a/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md b/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md index 2df01885f786..31ecf6891f26 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md @@ -125,7 +125,7 @@ Name | Type | Description | Default | Required ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ### output block diff --git a/docs/sources/flow/reference/components/otelcol.receiver.otlp.md b/docs/sources/flow/reference/components/otelcol.receiver.otlp.md index 92bc6e4a97bc..f0c609e2b521 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.otlp.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.otlp.md @@ -173,7 +173,7 @@ If `allowed_headers` includes `"*"`, all headers are permitted. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ### output block diff --git a/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md b/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md index f5554eee6b49..e23d4510b805 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md @@ -91,7 +91,7 @@ If `allowed_headers` includes `"*"`, all headers are permitted. ### debug_metrics block -{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" >}} +{{< docs/shared lookup="flow/reference/components/otelcol-debug-metrics-block.md" source="agent" version="" >}} ### output block From 387a6fd0288f2404f9be99e130edc57aa02cb4ae Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:21:15 +0200 Subject: [PATCH 12/24] Update docs/sources/flow/reference/components/otelcol.exporter.jaeger.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol.exporter.jaeger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md b/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md index 2fffdb05f9fc..5d2ad1c5088f 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.jaeger.md @@ -47,7 +47,7 @@ client > tls | [tls][] | Configures TLS for the gRPC client. | no client > keepalive | [keepalive][] | Configures keepalive settings for the gRPC client. | no sending_queue | [sending_queue][] | Configures batching of data before sending. | no retry_on_failure | [retry_on_failure][] | Configures retry mechanism for failed requests. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `client > tls` refers to a `tls` block defined inside a `client` block. From bdbd42059bac5f414461b42a8dc70d9151637f82 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:21:27 +0200 Subject: [PATCH 13/24] Update docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol.exporter.loadbalancing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md b/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md index 024874b508ed..6f27291898f8 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.loadbalancing.md @@ -82,7 +82,7 @@ protocol > otlp > client > tls | [tls][] | Configures TLS for the gRPC client. | protocol > otlp > client > keepalive | [keepalive][] | Configures keepalive settings for the gRPC client. | no protocol > otlp > queue | [queue][] | Configures batching of data before sending. | no protocol > otlp > retry | [retry][] | Configures retry mechanism for failed requests. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `resolver > static` refers to a `static` block defined inside a `resolver` block. From 1474782dcb643e161632e089c3b062d642d16c7d Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:21:41 +0200 Subject: [PATCH 14/24] Update docs/sources/flow/reference/components/otelcol.exporter.logging.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol.exporter.logging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.exporter.logging.md b/docs/sources/flow/reference/components/otelcol.exporter.logging.md index 1ed90bfd5423..100bde3fee5b 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.logging.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.logging.md @@ -45,7 +45,7 @@ The following blocks are supported inside the definition of Hierarchy | Block | Description | Required --------- | ----- | ----------- | -------- -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `client > tls` refers to a `tls` block defined inside a `client` block. From ad4771b26cea2885406403b4d0530458c3444e10 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:21:48 +0200 Subject: [PATCH 15/24] Update docs/sources/flow/reference/components/otelcol.exporter.otlp.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/flow/reference/components/otelcol.exporter.otlp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.exporter.otlp.md b/docs/sources/flow/reference/components/otelcol.exporter.otlp.md index f12d6784db34..53a9f3865dcc 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.otlp.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.otlp.md @@ -45,7 +45,7 @@ client > tls | [tls][] | Configures TLS for the gRPC client. | no client > keepalive | [keepalive][] | Configures keepalive settings for the gRPC client. | no sending_queue | [sending_queue][] | Configures batching of data before sending. | no retry_on_failure | [retry_on_failure][] | Configures retry mechanism for failed requests. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `client > tls` refers to a `tls` block defined inside a `client` block. From 929ce4bf40e67f8a7736b68da662da9126b896f9 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:22:02 +0200 Subject: [PATCH 16/24] Update docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol.exporter.otlphttp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md b/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md index f87eeb1dbf4e..b294e9ad63a7 100644 --- a/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md +++ b/docs/sources/flow/reference/components/otelcol.exporter.otlphttp.md @@ -50,7 +50,7 @@ client | [client][] | Configures the HTTP server to send telemetry dat client > tls | [tls][] | Configures TLS for the HTTP client. | no sending_queue | [sending_queue][] | Configures batching of data before sending. | no retry_on_failure | [retry_on_failure][] | Configures retry mechanism for failed requests. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no The `>` symbol indicates deeper levels of nesting. For example, `client > tls` refers to a `tls` block defined inside a `client` block. From 778ec08b41c99906703322a08e4773e867a35f41 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:22:12 +0200 Subject: [PATCH 17/24] Update docs/sources/flow/reference/components/otelcol.receiver.jaeger.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol.receiver.jaeger.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md b/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md index 243964b50f4f..e07d1ed1bce1 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.jaeger.md @@ -57,7 +57,7 @@ protocols > thrift_http > tls | [tls][] | Configures TLS for the Thrift HTTP ser protocols > thrift_http > cors | [cors][] | Configures CORS for the Thrift HTTP server. | no protocols > thrift_binary | [thrift_binary][] | Configures a Thrift binary UDP server to receive traces. | no protocols > thrift_compact | [thrift_compact][] | Configures a Thrift compact UDP server to receive traces. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no output | [output][] | Configures where to send received telemetry data. | yes The `>` symbol indicates deeper levels of nesting. For example, `protocols > From e793d6507b7352c0b8fbce18b2dadc9299fb21e9 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:22:27 +0200 Subject: [PATCH 18/24] Update docs/sources/flow/reference/components/otelcol.receiver.opencensus.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol.receiver.opencensus.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md b/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md index 31ecf6891f26..c32f3f4f92a0 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.opencensus.md @@ -67,7 +67,7 @@ tls | [tls][] | Configures TLS for the gRPC server. | no keepalive | [keepalive][] | Configures keepalive settings for the configured server. | no keepalive > server_parameters | [server_parameters][] | Server parameters used to configure keepalive settings. | no keepalive > enforcement_policy | [enforcement_policy][] | Enforcement policy for keepalive settings. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no output | [output][] | Configures where to send received telemetry data. | yes The `>` symbol indicates deeper levels of nesting. For example, `grpc > tls` From 6d04ee71bbe1f6a381b68934477ddf495828b83a Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:22:35 +0200 Subject: [PATCH 19/24] Update docs/sources/flow/reference/components/otelcol.receiver.otlp.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- docs/sources/flow/reference/components/otelcol.receiver.otlp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.receiver.otlp.md b/docs/sources/flow/reference/components/otelcol.receiver.otlp.md index f0c609e2b521..be0ac403f26b 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.otlp.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.otlp.md @@ -50,7 +50,7 @@ grpc > keepalive > enforcement_policy | [enforcement_policy][] | Enforcement pol http | [http][] | Configures the HTTP server to receive telemetry data. | no http > tls | [tls][] | Configures TLS for the HTTP server. | no http > cors | [cors][] | Configures CORS for the HTTP server. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no output | [output][] | Configures where to send received telemetry data. | yes The `>` symbol indicates deeper levels of nesting. For example, `grpc > tls` From d99b62dcbb5d08efdc555c6309967f1a3ebe7c5e Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:22:45 +0200 Subject: [PATCH 20/24] Update docs/sources/flow/reference/components/otelcol.receiver.zipkin.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol.receiver.zipkin.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md b/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md index e23d4510b805..0c4daf2beb3f 100644 --- a/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md +++ b/docs/sources/flow/reference/components/otelcol.receiver.zipkin.md @@ -49,7 +49,7 @@ Hierarchy | Block | Description | Required --------- | ----- | ----------- | -------- tls | [tls][] | Configures TLS for the HTTP server. | no cors | [cors][] | Configures CORS for the HTTP server. | no -debug_metrics | [debug_metrics][] | Configures the metrics which this component generates to monitor its state. | no +debug_metrics | [debug_metrics][] | Configures the metrics that this component generates to monitor its state. | no output | [output][] | Configures where to send received traces. | yes The `>` symbol indicates deeper levels of nesting. For example, `grpc > tls` From 93534feecaaff3cd2c3b51c930463838219f954d Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:22:56 +0200 Subject: [PATCH 21/24] Update docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol-debug-metrics-block.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md index 11e8eb3aecbf..033a0754e456 100644 --- a/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md +++ b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md @@ -5,7 +5,7 @@ canonical: https://grafana.com/docs/agent/latest/shared/flow/reference/component headless: true --- -The `debug_metrics` block configures the metrics which this component generates to monitor its state. +The `debug_metrics` block configures the metrics that this component generates to monitor its state. The following arguments are supported: From a95aafd7fa78266988fc9379fbf59ef7baea0444 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:23:10 +0200 Subject: [PATCH 22/24] Update docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> --- .../flow/reference/components/otelcol-debug-metrics-block.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md index 033a0754e456..416189914c89 100644 --- a/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md +++ b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md @@ -15,6 +15,6 @@ Name | Type | Description | Default | Required `disable_high_cardinality_metrics` is the Grafana Agent equivalent to the `telemetry.useOtelForInternalMetrics` feature gate in the OpenTelemetry -Collector. It removes attributes which could case high cardinality metrics. For +Collector. It removes attributes which could cause high cardinality metrics. For example, attributes with IP addresses and port numbers in metrics about HTTP and gRPC connections will be removed. From 93fbef25b1965b3913cd941b744fead652b79521 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:24:51 +0200 Subject: [PATCH 23/24] Update docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md Co-authored-by: Paulin Todev --- .../flow/reference/components/otelcol-debug-metrics-block.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md index 416189914c89..496bda9d3f35 100644 --- a/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md +++ b/docs/sources/shared/flow/reference/components/otelcol-debug-metrics-block.md @@ -14,7 +14,7 @@ Name | Type | Description | Default | Required `disable_high_cardinality_metrics` | `boolean` | Whether to disable certain high cardinality metrics. | `false` | no `disable_high_cardinality_metrics` is the Grafana Agent equivalent to the -`telemetry.useOtelForInternalMetrics` feature gate in the OpenTelemetry +`telemetry.disableHighCardinalityMetrics` feature gate in the OpenTelemetry Collector. It removes attributes which could cause high cardinality metrics. For example, attributes with IP addresses and port numbers in metrics about HTTP and gRPC connections will be removed. From bdc9fa287ef538d16e1167787eb1fb559e0c3d95 Mon Sep 17 00:00:00 2001 From: Gustaf Lindstedt Date: Fri, 1 Sep 2023 15:26:44 +0200 Subject: [PATCH 24/24] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28b13671a7b3..371420b5fad7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,9 @@ Main (unreleased) - Clustering: Allow advertise interfaces to be configurable. (@wildum) +- Added a `disable_high_cardinality_metrics` configuration flag to `otelcol` + exporters and receivers to switch high cardinality debug metrics off. (@glindstedt) + ### Other changes - Use Go 1.21.0 for builds. (@rfratto)