metrics: add count for number of upscale and downscale (#210)

Motivation
----------
Currently the wpa controlleur is only exposing a logs to inform of any upscale/downscale.
It would be very useful to have a metric on upscale/downscale in order to be able to visualize them.

see: https://github.com/DataDog/watermarkpodautoscaler/blob/9adcbcf5a9df4bb6b4370820dead417afa7c6932/controllers/watermarkpodautoscaler_controller.go#L379

Scope
------
Create two count metrics:
- `watermarkpodautoscaler.wpa_controller_upscale_replicas_total`: count the number of upscale done
- `watermarkpodautoscaler.wpa_controller_downscale_replicas_total`: count the number of down scale

controllers/metrics.go

@@ -43,6 +43,34 @@ var reasonValues = []string{downscaleCappingPromLabelVal, upscaleCappingPromLabe
 var extraPromLabels = strings.Fields(os.Getenv("DD_LABELS_AS_TAGS"))
 
 var (
+	upscale = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: subsystem,
+			Name:      "upscale_replicas_total",
+			Help:      "",
+		},
+		[]string{
+			wpaNamePromLabel,
+			wpaNamespacePromLabel,
+			resourceNamespacePromLabel,
+			resourceNamePromLabel,
+			resourceKindPromLabel,
+		},
+	)
+	downscale = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: subsystem,
+			Name:      "downscale_replicas_total",
+			Help:      "",
+		},
+		[]string{
+			wpaNamePromLabel,
+			wpaNamespacePromLabel,
+			resourceNamespacePromLabel,
+			resourceNamePromLabel,
+			resourceKindPromLabel,
+		},
+	)
 	value = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Subsystem: subsystem,
@@ -307,3 +335,13 @@ func cleanupAssociatedMetrics(wpa *datadoghqv1alpha1.WatermarkPodAutoscaler, onl
 		monitorNamespace:      wpa.Namespace,
 	})
 }
+
+func getPrometheusLabels(wpa *datadoghqv1alpha1.WatermarkPodAutoscaler) prometheus.Labels {
+	return prometheus.Labels{
+		wpaNamePromLabel:           wpa.Name,
+		wpaNamespacePromLabel:      wpa.Namespace,
+		resourceNamePromLabel:      wpa.Spec.ScaleTargetRef.Name,
+		resourceNamespacePromLabel: wpa.Namespace,
+		resourceKindPromLabel:      wpa.Spec.ScaleTargetRef.Kind,
+	}
+}

controllers/watermarkpodautoscaler_controller.go

@@ -377,6 +377,11 @@ func (r *WatermarkPodAutoscalerReconciler) reconcileWPA(ctx context.Context, log
 		r.eventRecorder.Eventf(wpa, corev1.EventTypeNormal, datadoghqv1alpha1.ReasonScaling, fmt.Sprintf("New size: %d; reason: %s", desiredReplicas, rescaleReason))
 
 		logger.Info("Successful rescale", "currentReplicas", currentReplicas, "desiredReplicas", desiredReplicas, "rescaleReason", rescaleReason)
+		if currentReplicas < desiredReplicas {
+			upscale.With(getPrometheusLabels(wpa)).Add(float64(desiredReplicas - currentReplicas))
+		} else {
+			downscale.With(getPrometheusLabels(wpa)).Add(float64(currentReplicas - desiredReplicas))
+		}
 	} else {
 		if r.Options.SkipNotScalingEvents {
 			setCondition(wpa, autoscalingv2.ScalingActive, corev1.ConditionTrue, datadoghqv1alpha1.ConditionReasonNotScaling, "the WPA was able to successfully calculate a replica count and decided not to scale %s to %d (last scale time was %v )", reference, desiredReplicas, wpa.Status.LastScaleTime)