From 59f17fd23f84fe367f6e04a98aaa01a728e0eaa7 Mon Sep 17 00:00:00 2001 From: Greg Eales <0x006ea1e5@gmail.com> Date: Wed, 15 May 2024 14:51:32 +0100 Subject: [PATCH] Add reason dimension to exporter and receiver failure metrics --- exporter/exporterhelper/obsexporter.go | 16 +++++++++++----- receiver/receiverhelper/obsreport.go | 12 +++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/exporter/exporterhelper/obsexporter.go b/exporter/exporterhelper/obsexporter.go index e3a78c34b04..84dac862f24 100644 --- a/exporter/exporterhelper/obsexporter.go +++ b/exporter/exporterhelper/obsexporter.go @@ -11,6 +11,7 @@ import ( "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "go.uber.org/zap" + "google.golang.org/grpc/status" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/configtelemetry" @@ -70,7 +71,7 @@ func (or *ObsReport) StartTracesOp(ctx context.Context) context.Context { // EndTracesOp completes the export operation that was started with StartTracesOp. func (or *ObsReport) EndTracesOp(ctx context.Context, numSpans int, err error) { numSent, numFailedToSend := toNumItems(numSpans, err) - or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeTraces, numSent, numFailedToSend) + or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeTraces, numSent, numFailedToSend, err) endSpan(ctx, err, numSent, numFailedToSend, obsmetrics.SentSpansKey, obsmetrics.FailedToSendSpansKey) } @@ -85,7 +86,7 @@ func (or *ObsReport) StartMetricsOp(ctx context.Context) context.Context { // StartMetricsOp. func (or *ObsReport) EndMetricsOp(ctx context.Context, numMetricPoints int, err error) { numSent, numFailedToSend := toNumItems(numMetricPoints, err) - or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeMetrics, numSent, numFailedToSend) + or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeMetrics, numSent, numFailedToSend, err) endSpan(ctx, err, numSent, numFailedToSend, obsmetrics.SentMetricPointsKey, obsmetrics.FailedToSendMetricPointsKey) } @@ -99,7 +100,7 @@ func (or *ObsReport) StartLogsOp(ctx context.Context) context.Context { // EndLogsOp completes the export operation that was started with StartLogsOp. func (or *ObsReport) EndLogsOp(ctx context.Context, numLogRecords int, err error) { numSent, numFailedToSend := toNumItems(numLogRecords, err) - or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeLogs, numSent, numFailedToSend) + or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeLogs, numSent, numFailedToSend, err) endSpan(ctx, err, numSent, numFailedToSend, obsmetrics.SentLogRecordsKey, obsmetrics.FailedToSendLogRecordsKey) } @@ -111,7 +112,7 @@ func (or *ObsReport) startOp(ctx context.Context, operationSuffix string) contex return ctx } -func (or *ObsReport) recordMetrics(ctx context.Context, dataType component.DataType, sent, failed int64) { +func (or *ObsReport) recordMetrics(ctx context.Context, dataType component.DataType, sent, failed int64, err error) { if or.level == configtelemetry.LevelNone { return } @@ -129,7 +130,12 @@ func (or *ObsReport) recordMetrics(ctx context.Context, dataType component.DataT } sentMeasure.Add(ctx, sent, metric.WithAttributes(or.otelAttrs...)) - failedMeasure.Add(ctx, failed, metric.WithAttributes(or.otelAttrs...)) + if err != nil && or.level == configtelemetry.LevelDetailed { + failedMeasure.Add(ctx, failed, metric.WithAttributes( + append(or.otelAttrs, attribute.String("reason", status.Convert(err).Code().String()))...)) + } else { + failedMeasure.Add(ctx, failed, metric.WithAttributes(or.otelAttrs...)) + } } func endSpan(ctx context.Context, err error, numSent, numFailedToSend int64, sentItemsKey, failedToSendItemsKey string) { diff --git a/receiver/receiverhelper/obsreport.go b/receiver/receiverhelper/obsreport.go index 18ff7556454..abb4deaba31 100644 --- a/receiver/receiverhelper/obsreport.go +++ b/receiver/receiverhelper/obsreport.go @@ -7,6 +7,7 @@ package receiverhelper // import "go.opentelemetry.io/collector/receiver/receive import ( "context" + "google.golang.org/grpc/status" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" @@ -170,7 +171,7 @@ func (rec *ObsReport) endOp( span := trace.SpanFromContext(receiverCtx) if rec.level != configtelemetry.LevelNone { - rec.recordMetrics(receiverCtx, dataType, numAccepted, numRefused) + rec.recordMetrics(receiverCtx, dataType, numAccepted, numRefused, err) } // end span according to errors @@ -200,7 +201,7 @@ func (rec *ObsReport) endOp( span.End() } -func (rec *ObsReport) recordMetrics(receiverCtx context.Context, dataType component.DataType, numAccepted, numRefused int) { +func (rec *ObsReport) recordMetrics(receiverCtx context.Context, dataType component.DataType, numAccepted, numRefused int, err error) { var acceptedMeasure, refusedMeasure metric.Int64Counter switch dataType { case component.DataTypeTraces: @@ -215,5 +216,10 @@ func (rec *ObsReport) recordMetrics(receiverCtx context.Context, dataType compon } acceptedMeasure.Add(receiverCtx, int64(numAccepted), metric.WithAttributes(rec.otelAttrs...)) - refusedMeasure.Add(receiverCtx, int64(numRefused), metric.WithAttributes(rec.otelAttrs...)) + if err != nil && rec.level == configtelemetry.LevelDetailed { + refusedMeasure.Add(receiverCtx, int64(numRefused), metric.WithAttributes( + append(rec.otelAttrs, attribute.String("reason", status.Convert(err).Code().String()))...)) + } else { + refusedMeasure.Add(receiverCtx, int64(numRefused), metric.WithAttributes(rec.otelAttrs...)) + } }