From 47a2db5ccda7b96e42876b85d5d4c42fd1fd928a Mon Sep 17 00:00:00 2001
From: Greg Eales <0x006ea1e5@gmail.com>
Date: Wed, 15 May 2024 14:51:32 +0100
Subject: [PATCH] Add reason dimension to exporter and receiver failure metrics

---
 exporter/exporterhelper/obsexporter.go | 16 +++++++++++-----
 receiver/receiverhelper/obsreport.go   | 12 +++++++++---
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/exporter/exporterhelper/obsexporter.go b/exporter/exporterhelper/obsexporter.go
index e3a78c34b04..84dac862f24 100644
--- a/exporter/exporterhelper/obsexporter.go
+++ b/exporter/exporterhelper/obsexporter.go
@@ -11,6 +11,7 @@ import (
 	"go.opentelemetry.io/otel/metric"
 	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
+	"google.golang.org/grpc/status"
 
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config/configtelemetry"
@@ -70,7 +71,7 @@ func (or *ObsReport) StartTracesOp(ctx context.Context) context.Context {
 // EndTracesOp completes the export operation that was started with StartTracesOp.
 func (or *ObsReport) EndTracesOp(ctx context.Context, numSpans int, err error) {
 	numSent, numFailedToSend := toNumItems(numSpans, err)
-	or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeTraces, numSent, numFailedToSend)
+	or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeTraces, numSent, numFailedToSend, err)
 	endSpan(ctx, err, numSent, numFailedToSend, obsmetrics.SentSpansKey, obsmetrics.FailedToSendSpansKey)
 }
 
@@ -85,7 +86,7 @@ func (or *ObsReport) StartMetricsOp(ctx context.Context) context.Context {
 // StartMetricsOp.
 func (or *ObsReport) EndMetricsOp(ctx context.Context, numMetricPoints int, err error) {
 	numSent, numFailedToSend := toNumItems(numMetricPoints, err)
-	or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeMetrics, numSent, numFailedToSend)
+	or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeMetrics, numSent, numFailedToSend, err)
 	endSpan(ctx, err, numSent, numFailedToSend, obsmetrics.SentMetricPointsKey, obsmetrics.FailedToSendMetricPointsKey)
 }
 
@@ -99,7 +100,7 @@ func (or *ObsReport) StartLogsOp(ctx context.Context) context.Context {
 // EndLogsOp completes the export operation that was started with StartLogsOp.
 func (or *ObsReport) EndLogsOp(ctx context.Context, numLogRecords int, err error) {
 	numSent, numFailedToSend := toNumItems(numLogRecords, err)
-	or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeLogs, numSent, numFailedToSend)
+	or.recordMetrics(noCancellationContext{Context: ctx}, component.DataTypeLogs, numSent, numFailedToSend, err)
 	endSpan(ctx, err, numSent, numFailedToSend, obsmetrics.SentLogRecordsKey, obsmetrics.FailedToSendLogRecordsKey)
 }
 
@@ -111,7 +112,7 @@ func (or *ObsReport) startOp(ctx context.Context, operationSuffix string) contex
 	return ctx
 }
 
-func (or *ObsReport) recordMetrics(ctx context.Context, dataType component.DataType, sent, failed int64) {
+func (or *ObsReport) recordMetrics(ctx context.Context, dataType component.DataType, sent, failed int64, err error) {
 	if or.level == configtelemetry.LevelNone {
 		return
 	}
@@ -129,7 +130,12 @@ func (or *ObsReport) recordMetrics(ctx context.Context, dataType component.DataT
 	}
 
 	sentMeasure.Add(ctx, sent, metric.WithAttributes(or.otelAttrs...))
-	failedMeasure.Add(ctx, failed, metric.WithAttributes(or.otelAttrs...))
+	if err != nil  && or.level == configtelemetry.LevelDetailed {
+		failedMeasure.Add(ctx, failed, metric.WithAttributes(
+			append(or.otelAttrs, attribute.String("reason", status.Convert(err).Code().String()))...))
+	} else {
+		failedMeasure.Add(ctx, failed, metric.WithAttributes(or.otelAttrs...))
+	}
 }
 
 func endSpan(ctx context.Context, err error, numSent, numFailedToSend int64, sentItemsKey, failedToSendItemsKey string) {
diff --git a/receiver/receiverhelper/obsreport.go b/receiver/receiverhelper/obsreport.go
index 18ff7556454..abb4deaba31 100644
--- a/receiver/receiverhelper/obsreport.go
+++ b/receiver/receiverhelper/obsreport.go
@@ -7,6 +7,7 @@ package receiverhelper // import "go.opentelemetry.io/collector/receiver/receive
 
 import (
 	"context"
+	"google.golang.org/grpc/status"
 
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
@@ -170,7 +171,7 @@ func (rec *ObsReport) endOp(
 	span := trace.SpanFromContext(receiverCtx)
 
 	if rec.level != configtelemetry.LevelNone {
-		rec.recordMetrics(receiverCtx, dataType, numAccepted, numRefused)
+		rec.recordMetrics(receiverCtx, dataType, numAccepted, numRefused, err)
 	}
 
 	// end span according to errors
@@ -200,7 +201,7 @@ func (rec *ObsReport) endOp(
 	span.End()
 }
 
-func (rec *ObsReport) recordMetrics(receiverCtx context.Context, dataType component.DataType, numAccepted, numRefused int) {
+func (rec *ObsReport) recordMetrics(receiverCtx context.Context, dataType component.DataType, numAccepted, numRefused int, err error) {
 	var acceptedMeasure, refusedMeasure metric.Int64Counter
 	switch dataType {
 	case component.DataTypeTraces:
@@ -215,5 +216,10 @@ func (rec *ObsReport) recordMetrics(receiverCtx context.Context, dataType compon
 	}
 
 	acceptedMeasure.Add(receiverCtx, int64(numAccepted), metric.WithAttributes(rec.otelAttrs...))
-	refusedMeasure.Add(receiverCtx, int64(numRefused), metric.WithAttributes(rec.otelAttrs...))
+	if err != nil && rec.level == configtelemetry.LevelDetailed {
+		refusedMeasure.Add(receiverCtx, int64(numRefused), metric.WithAttributes(
+			append(rec.otelAttrs, attribute.String("reason", status.Convert(err).Code().String()))...))
+	} else {
+		refusedMeasure.Add(receiverCtx, int64(numRefused), metric.WithAttributes(rec.otelAttrs...))
+	}
 }