add mapi_machinehealthcheck_remediation_success_total metric

docs/dev/metrics.md

@@ -118,14 +118,21 @@ When using MachineHealthChecks, metrics are available from the `machine-api-cont
 default metrics port(`8083`) for the `machine-healthcheck-controller` container.
 
 The `mapi_machinehealthcheck_nodes_covered` metric describes the number of Nodes that are currently
-being monitored by `machine-healthcheck-controller`. The `name` label in this metric refers to the
-name of the MachineHealthCheck that is being reported. The `namespace` label refers to the owning
-namespace of the MachineHealthCheck.
+being monitored by `machine-healthcheck-controller`.
+
+The `mapi_machinehealthcheck_remediation_success_total` metric gives a total count of the successful
+remediation performed by a MachineHealthCheck.
+
+The `name` label in these metric refers to the name of the MachineHealthCheck that is being reported.
+The `namespace` label refers to the owning namespace of the MachineHealthCheck.
 
 **Sample metrics**
 ```
 # HELP mapi_machinehealthcheck_nodes_covered Number of nodes covered by MachineHealthChecks
 # TYPE mapi_machinehealthcheck_nodes_covered gauge
 mapi_machinehealthcheck_nodes_covered{name="machine-api-termination-handler",namespace="openshift-machine-api"} 0
 mapi_machinehealthcheck_nodes_covered{name="mhc-1",namespace="openshift-machine-api"} 1
+# HELP mapi_machinehealthcheck_remediation_success_total Number of successful remediations performed by MachineHealthChecks
+# TYPE mapi_machinehealthcheck_remediation_success_total counter
+mapi_machinehealthcheck_remediation_success_total{name="mhc-1",namespace="openshift-machine-api"} 1
 ```

pkg/controller/machinehealthcheck/machinehealthcheck_controller.go

@@ -541,6 +541,8 @@ func (t *target) remediate(r *ReconcileMachineHealthCheck) error {
 		"Machine %v has been remediated by requesting to delete Machine object",
 		t.string(),
 	)
+	metrics.ObserveMachineHealthCheckRemediationSuccess(t.MHC.Name, t.MHC.Namespace)
+
 	return nil
 }
 

pkg/metrics/machinehealthcheck.go

@@ -32,10 +32,21 @@ var (
 			Help: "Number of nodes covered by MachineHealthChecks",
 		}, []string{"name", "namespace"},
 	)
+
+	// MachineHealthCheckRemediationSuccessTotal is a Prometheus metric, which reports the number of successful remediations by MachineHealthChecks
+	MachineHealthCheckRemediationSuccessTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mapi_machinehealthcheck_remediation_success_total",
+			Help: "Number of successful remediations performed by MachineHealthChecks",
+		}, []string{"name", "namespace"},
+	)
 )
 
 func InitializeMachineHealthCheckMetrics() {
-	metrics.Registry.MustRegister(MachineHealthCheckNodesCovered)
+	metrics.Registry.MustRegister(
+		MachineHealthCheckNodesCovered,
+		MachineHealthCheckRemediationSuccessTotal,
+	)
 }
 
 func DeleteMachineHealthCheckNodesCovered(name string, namespace string) {
@@ -51,3 +62,10 @@ func ObserveMachineHealthCheckNodesCovered(name string, namespace string, count
 		"namespace": namespace,
 	}).Set(float64(count))
 }
+
+func ObserveMachineHealthCheckRemediationSuccess(name string, namespace string) {
+	MachineHealthCheckRemediationSuccessTotal.With(prometheus.Labels{
+		"name":      name,
+		"namespace": namespace,
+	}).Inc()
+}