Add per tenant bytes counter (#331)

* Add per tenant bytes counter

Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com>

* Add changelog

Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com>

* Rename metric

Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com>

* Add metric after error check

Signed-off-by: Daniel González Lopes <danielgonzalezlopes@gmail.com>

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [ENHANCEMENT] Add warnings for suspect configs. [#294](https://github.com/grafana/tempo/pull/294)
 * [ENHANCEMENT] Add command line flags for s3 credentials. [#308](https://github.com/grafana/tempo/pull/308)
 * [ENHANCEMENT] Support multiple authentication methods for S3 (IRSA, IAM role, static). [#320](https://github.com/grafana/tempo/pull/320)
+* [ENHANCEMENT] Add  per tenant bytes counter. [#331](https://github.com/grafana/tempo/pull/331)
 * [BUGFIX] S3 multi-part upload errors [#306](https://github.com/grafana/tempo/pull/325)
 * [BUGFIX] Increase Prometheus `notfound` metric on tempo-vulture. [#301](https://github.com/grafana/tempo/pull/301)
 * [BUGFIX] Return 404 if searching for a tenant id that does not exist in the backend. [#321](https://github.com/grafana/tempo/pull/321)

modules/ingester/instance.go

@@ -33,6 +33,11 @@ var (
 		Name:      "ingester_traces_created_total",
 		Help:      "The total number of traces created per tenant.",
 	}, []string{"tenant"})
+	metricBytesWrittenTotal = promauto.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "tempo",
+		Name:      "ingester_bytes_written_total",
+		Help:      "The total bytes written per tenant.",
+	}, []string{"tenant"})
 	metricBlocksClearedTotal = promauto.NewCounter(prometheus.CounterOpts{
 		Namespace: "tempo",
 		Name:      "ingester_blocks_cleared_total",
@@ -52,6 +57,7 @@ type instance struct {
 
 	instanceID         string
 	tracesCreatedTotal prometheus.Counter
+	bytesWrittenTotal  prometheus.Counter
 	limiter            *Limiter
 	wal                *tempodb_wal.WAL
 }
@@ -62,6 +68,7 @@ func newInstance(instanceID string, limiter *Limiter, wal *tempodb_wal.WAL) (*in
 
 		instanceID:         instanceID,
 		tracesCreatedTotal: metricTracesCreatedTotal.WithLabelValues(instanceID),
+		bytesWrittenTotal:  metricBytesWrittenTotal.WithLabelValues(instanceID),
 		limiter:            limiter,
 		wal:                wal,
 	}
@@ -111,12 +118,11 @@ func (i *instance) CutCompleteTraces(cutoff time.Duration, immediate bool) error
 			if err != nil {
 				return err
 			}
-
 			err = i.headBlock.Write(trace.traceID, out)
 			if err != nil {
 				return err
 			}
-
+			i.bytesWrittenTotal.Add(float64(len(out)))
 			delete(i.traces, key)
 		}
 	}