add mapi_machinehealthcheck_remediation_success_total metric

This change add the metric as well as a helper function to increase the
count, and update the metric doc to reflect the new metric.

docs/dev/metrics.md

@@ -115,16 +115,23 @@ mapi_instance_delete_failed{name=machine-3,namespace=openshift-machine-api,reaso
 ## Metrics about MachineHealthCheck resources
 
 When using MachineHealthChecks, metrics are available from the `machine-api-controllers` Pod on the
-default metrics port(`8083`) for the `machine-healthcheck-controller` container. 
+default metrics port(`8083`) for the `machine-healthcheck-controller` container.
 
 The `mapi_machinehealthcheck_nodes_covered` metric describes the number of Nodes that are currently
 being monitored by `machine-healthcheck-controller`. The `name` label in this metric refers to the
 name of the MachineHealthCheck that is being reported.
 
+The `mapi_machinehealthcheck_remediation_success_total` metric gives a total count of the successful
+remediation performed by a MachineHealthCheck. The `name` label in this metric refers to the name
+of the MachineHealthCheck that is being reported.
+
 **Sample metrics**
 ```
 # HELP mapi_machinehealthcheck_nodes_covered Number of nodes covered by MachineHealthChecks
 # TYPE mapi_machinehealthcheck_nodes_covered gauge
 mapi_machinehealthcheck_nodes_covered{name="machine-api-termination-handler"} 0
 mapi_machinehealthcheck_nodes_covered{name="mhc-1"} 1
+# HELP mapi_machinehealthcheck_remediation_success_total Number of successful remediations performed by MachineHealthChecks
+# TYPE mapi_machinehealthcheck_remediation_success_total counter
+mapi_machinehealthcheck_remediation_success_total{name="mhc-1"} 1
 ```

pkg/controller/machinehealthcheck/machinehealthcheck_controller.go

@@ -492,6 +492,8 @@ func (t *target) remediate(r *ReconcileMachineHealthCheck) error {
 		"Machine %v has been remediated by requesting to delete Machine object",
 		t.string(),
 	)
+	metrics.RegisterMachineHealthCheckRemediationSuccess(t.MHC.Name)
+
 	return nil
 }
 

pkg/metrics/machinehealthcheck.go

@@ -32,10 +32,21 @@ var (
 			Help: "Number of nodes covered by MachineHealthChecks",
 		}, []string{"name"},
 	)
+
+	// MachineHealthCheckRemediationSuccessTotal is a Prometheus metric, which reports the number of successful remediations by MachineHealthChecks
+	MachineHealthCheckRemediationSuccessTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "mapi_machinehealthcheck_remediation_success_total",
+			Help: "Number of successful remediations performed by MachineHealthChecks",
+		}, []string{"name"},
+	)
 )
 
 func InitializeMachineHealthCheckMetrics() {
-	metrics.Registry.MustRegister(MachineHealthCheckNodesCovered)
+	metrics.Registry.MustRegister(
+		MachineHealthCheckNodesCovered,
+		MachineHealthCheckRemediationSuccessTotal,
+	)
 }
 
 func DeleteMachineHealthCheckNodesCovered(name string) {
@@ -45,3 +56,7 @@ func DeleteMachineHealthCheckNodesCovered(name string) {
 func RegisterMachineHealthCheckNodesCovered(name string, count int) {
 	MachineHealthCheckNodesCovered.With(prometheus.Labels{"name": name}).Set(float64(count))
 }
+
+func RegisterMachineHealthCheckRemediationSuccess(name string) {
+	MachineHealthCheckRemediationSuccessTotal.With(prometheus.Labels{"name": name}).Inc()
+}