Namespace tag the disruptions.gauge metric (#847)

* Namespace tag the disruptions.gauge metric Co-authored-by: zhiyanfoo <zhiyanfoo@gmail.com>
DataDog · Mar 15, 2024 · 4d98cfd · 4d98cfd
1 parent 082d863
commit 4d98cfd
Show file tree

Hide file tree

Showing 5 changed files with 53 additions and 16 deletions.
diff --git a/controllers/disruption_controller.go b/controllers/disruption_controller.go
@@ -1025,6 +1025,8 @@ func (r *DisruptionReconciler) ReportMetrics(ctx context.Context) {
 			continue
 		}
 
+		namespaces := map[string]int{}
+
 		// check for stuck durations, count chaos pods, and track ongoing disruption duration
 		for _, d := range l.Items {
 			if d.Status.IsStuckOnRemoval {
@@ -1043,15 +1045,25 @@ func (r *DisruptionReconciler) ReportMetrics(ctx context.Context) {
 			chaosPodsCount += len(chaosPods)
 
 			r.handleMetricSinkError(r.MetricsSink.MetricDisruptionOngoingDuration(time.Since(d.ObjectMeta.CreationTimestamp.Time), []string{"disruptionName:" + d.Name, "namespace:" + d.Namespace}))
+
+			namespaces[d.Namespace]++
 		}
 
 		// send metrics
 		if err := r.MetricsSink.MetricStuckOnRemovalGauge(float64(stuckOnRemoval)); err != nil {
 			r.BaseLog.Errorw("error sending stuck_on_removal_total metric", "error", err)
 		}
 
-		if err := r.MetricsSink.MetricDisruptionsGauge(float64(len(l.Items))); err != nil {
-			r.BaseLog.Errorw("error sending disruptions.gauge metric", "error", err)
+		if len(namespaces) > 0 {
+			for namespace, count := range namespaces {
+				if err := r.MetricsSink.MetricDisruptionsGauge(float64(count), []string{fmt.Sprintf("namespace:%s", namespace)}); err != nil {
+					r.BaseLog.Errorw("error sending disruptions.gauge metric", "error", err)
+				}
+			}
+		} else {
+			if err := r.MetricsSink.MetricDisruptionsGauge(float64(len(l.Items)), []string{}); err != nil {
+				r.BaseLog.Errorw("error sending disruptions.gauge metric", "error", err)
+			}
 		}
 
 		if err := r.MetricsSink.MetricPodsGauge(float64(chaosPodsCount)); err != nil {

diff --git a/o11y/metrics/datadog/datadog.go b/o11y/metrics/datadog/datadog.go
@@ -164,8 +164,8 @@ func (d Sink) MetricStuckOnRemovalGauge(gauge float64) error {
 }
 
 // MetricDisruptionsGauge sends the disruptions.gauge metric counting ongoing disruptions
-func (d Sink) MetricDisruptionsGauge(gauge float64) error {
-	return d.client.Gauge(d.prefix+"disruptions.gauge", gauge, []string{}, 1)
+func (d Sink) MetricDisruptionsGauge(gauge float64, tags []string) error {
+	return d.client.Gauge(d.prefix+"disruptions.gauge", gauge, tags, 1)
 }
 
 // MetricDisruptionsCount counts finished disruptions, and tags the disruption kind

diff --git a/o11y/metrics/metrics.go b/o11y/metrics/metrics.go
@@ -34,7 +34,7 @@ type Sink interface {
 	MetricDisruptionOngoingDuration(duration time.Duration, tags []string) error
 	MetricStuckOnRemoval(tags []string) error
 	MetricStuckOnRemovalGauge(gauge float64) error
-	MetricDisruptionsGauge(gauge float64) error
+	MetricDisruptionsGauge(gauge float64, tags []string) error
 	MetricDisruptionsCount(kind chaostypes.DisruptionKindName, tags []string) error
 	MetricSelectorCacheGauge(gauge float64) error
 	MetricWatcherCalls(tags []string) error

diff --git a/o11y/metrics/noop/noop.go b/o11y/metrics/noop/noop.go
@@ -133,7 +133,7 @@ func (n Sink) MetricStuckOnRemovalGauge(gauge float64) error {
 }
 
 // MetricDisruptionsGauge sends disruptions.gauge metric
-func (n Sink) MetricDisruptionsGauge(gauge float64) error {
+func (n Sink) MetricDisruptionsGauge(gauge float64, tags []string) error {
 	n.log.Debugf("NOOP: MetricDisruptionsGauge %f\n", gauge)
 
 	return nil

diff --git a/o11y/metrics/sink_mock.go b/o11y/metrics/sink_mock.go