provide configuration to include custom node and pod labels on metrics

README.md

@@ -53,6 +53,7 @@ Table of Contents
 - [High Availability](#high-availability)
   - [Configure HA Mode](#configure-ha-mode)
 - [Metrics](#metrics)
+- [Metrics Config](#metrics-config)
 - [Compatibility Matrix](#compatibility-matrix)
 - [Getting Involved and Contributing](#getting-involved-and-contributing)
   - [Communicating With Contributors](#communicating-with-contributors)
@@ -136,6 +137,7 @@ The policy includes a common configuration that applies to all the strategies:
 | `maxNoOfPodsToEvictPerNode` | `nil` | maximum number of pods evicted from each node (summed through all strategies) |
 | `maxNoOfPodsToEvictPerNamespace` | `nil` | maximum number of pods evicted from each namespace (summed through all strategies) |
 | `evictFailedBarePods` | `false` | allow eviction of pods without owner references and in failed phase |
+| `metricsConfig` | `nil` | configuration to instruct Descheduler's metrics to include node and pod labels |
 
 As part of the policy, the parameters associated with each strategy can be configured.
 See each strategy for details on available parameters.
@@ -151,6 +153,7 @@ evictLocalStoragePods: true
 evictSystemCriticalPods: true
 maxNoOfPodsToEvictPerNode: 40
 ignorePvcPods: false
+metricsConfig: {}
 strategies:
   ...
 ```
@@ -821,6 +824,29 @@ To get best results from HA mode some additional configurations might require:
 The metrics are served through https://localhost:10258/metrics by default.
 The address and port can be changed by setting `--binding-address` and `--secure-port` flags.
 
+## Metrics Config
+Eviction metrics can be enriched by including node and pod labels.
+
+| name	| type	| description |
+|-------|-------|----------------|
+| `nodeLabels` | list(string) | list of node labels to include in metrics |
+| `podLabels` | list(string) | list of pod labels to include in metrics |
+
+```yaml
+apiVersion: "descheduler/v1alpha1"
+kind: "DeschedulerPolicy"
+metricsConfig:
+  nodeLabels:
+  - "topology.kubernetes.io/zone"
+  podLabels:
+  - "app.kubernetes.io/name"
+```
+
+In order to conform to [metrics names and labels data model](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels),
+all labels will be converted to only contain ASCII letters, numbers, and underscores.
+In other words, node label `topology.kubernetes.io/zone` will become metric label `topology_kubernetes_io_zone`.
+The metric value, however, will be left untouched.
+
 ## Compatibility Matrix
 The below compatibility matrix shows the k8s client package(client-go, apimachinery, etc) versions that descheduler
 is compiled with. At this time descheduler does not have a hard dependency to a specific k8s release. However a

examples/custom-metrics-labels.yaml

@@ -0,0 +1,10 @@
+apiVersion: "descheduler/v1alpha1"
+kind: "DeschedulerPolicy"
+metricsConfig:
+  nodeLabels:
+  - "topology.kubernetes.io/zone"
+  podLabels:
+  - "app.kubernetes.io/name"
+strategies:
+  "RemovePodsViolatingTopologySpreadConstraint":
+     enabled: true

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	github.com/client9/misspell v0.3.4
+	github.com/prometheus/client_golang v1.12.1
 	github.com/spf13/cobra v1.4.0
 	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.24.0
@@ -68,7 +69,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect

metrics/metrics.go

@@ -17,6 +17,7 @@ limitations under the License.
 package metrics
 
 import (
+	"regexp"
 	"sync"
 
 	"k8s.io/component-base/metrics"
@@ -30,14 +31,6 @@ const (
 )
 
 var (
-	PodsEvicted = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Subsystem:      DeschedulerSubsystem,
-			Name:           "pods_evicted",
-			Help:           "Number of evicted pods, by the result, by the strategy, by the namespace, by the node name. 'error' result means a pod could not be evicted",
-			StabilityLevel: metrics.ALPHA,
-		}, []string{"result", "strategy", "namespace", "node"})
-
 	buildInfo = metrics.NewGauge(
 		&metrics.GaugeOpts{
 			Subsystem:      DeschedulerSubsystem,
@@ -48,25 +41,46 @@ var (
 		},
 	)
 
-	metricsList = []metrics.Registerable{
-		PodsEvicted,
-		buildInfo,
-	}
+	metricsNameRegex, _ = regexp.Compile("[^a-zA-Z0-9_]+")
+
+	PodsEvicted *metrics.CounterVec
 )
 
 var registerMetrics sync.Once
 
 // Register all metrics.
-func Register() {
+func Register(customLabels []string) {
 	// Register the metrics.
 	registerMetrics.Do(func() {
-		RegisterMetrics(metricsList...)
+		metrics := initialize(customLabels)
+		for _, metric := range metrics {
+			legacyregistry.MustRegister(metric)
+		}
 	})
 }
 
-// RegisterMetrics registers a list of metrics.
-func RegisterMetrics(extraMetrics ...metrics.Registerable) {
-	for _, metric := range extraMetrics {
-		legacyregistry.MustRegister(metric)
+// ensure that labels conform to https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels
+func ConvertToMetricLabel(label string) string {
+	return metricsNameRegex.ReplaceAllString(label, "_")
+}
+
+func initialize(customLabels []string) []metrics.Registerable {
+	labels := []string{"result", "strategy", "namespace", "node"}
+	for _, customLabel := range customLabels {
+		labels = append(labels, ConvertToMetricLabel(customLabel))
+	}
+
+	PodsEvicted = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Subsystem:      DeschedulerSubsystem,
+			Name:           "pods_evicted",
+			Help:           "Number of evicted pods, by the result, by the strategy, by the namespace, by the node name. 'error' result means a pod could not be evicted",
+			StabilityLevel: metrics.ALPHA,
+		},
+		labels,
+	)
+
+	return []metrics.Registerable{
+		PodsEvicted,
 	}
 }

pkg/api/types.go

@@ -49,6 +49,14 @@ type DeschedulerPolicy struct {
 
 	// MaxNoOfPodsToEvictPerNamespace restricts maximum of pods to be evicted per namespace.
 	MaxNoOfPodsToEvictPerNamespace *uint
+
+	// MetricsConfig to control information included in metrics
+	MetricsConfig *MetricsConfig
+}
+
+type MetricsConfig struct {
+	NodeLabels []string
+	PodLabels  []string
 }
 
 type StrategyName string

pkg/api/v1alpha1/types.go

@@ -49,6 +49,14 @@ type DeschedulerPolicy struct {
 
 	// MaxNoOfPodsToEvictPerNamespace restricts maximum of pods to be evicted per namespace.
 	MaxNoOfPodsToEvictPerNamespace *int `json:"maxNoOfPodsToEvictPerNamespace,omitempty"`
+
+	// MetricsConfig to control information included in metrics
+	MetricsConfig *MetricsConfig `json:"metricsConfig"`
+}
+
+type MetricsConfig struct {
+	NodeLabels []string `json:"nodeLabels"`
+	PodLabels  []string `json:"podLabels"`
 }
 
 type StrategyName string

pkg/descheduler/descheduler.go

@@ -19,6 +19,7 @@ package descheduler
 import (
 	"context"
 	"fmt"
+	"sigs.k8s.io/descheduler/metrics"
 
 	v1 "k8s.io/api/core/v1"
 	policy "k8s.io/api/policy/v1beta1"
@@ -36,7 +37,6 @@ import (
 	schedulingv1informers "k8s.io/client-go/informers/scheduling/v1"
 
 	"sigs.k8s.io/descheduler/cmd/descheduler/app/options"
-	"sigs.k8s.io/descheduler/metrics"
 	"sigs.k8s.io/descheduler/pkg/api"
 	"sigs.k8s.io/descheduler/pkg/descheduler/client"
 	"sigs.k8s.io/descheduler/pkg/descheduler/evictions"
@@ -49,8 +49,6 @@ import (
 )
 
 func Run(ctx context.Context, rs *options.DeschedulerServer) error {
-	metrics.Register()
-
 	rsclient, err := client.CreateClient(rs.KubeconfigFile)
 	if err != nil {
 		return err
@@ -276,7 +274,20 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 			podEvictorClient = rs.Client
 		}
 
+		if !rs.DisableMetrics {
+			klog.V(3).Infof("Registering metrics")
+
+			var customLabels []string
+			if deschedulerPolicy.MetricsConfig != nil {
+				customLabels = append(customLabels, deschedulerPolicy.MetricsConfig.NodeLabels...)
+				customLabels = append(customLabels, deschedulerPolicy.MetricsConfig.PodLabels...)
+			}
+
+			metrics.Register(customLabels)
+		}
+
 		klog.V(3).Infof("Building a pod evictor")
+
 		podEvictor := evictions.NewPodEvictor(
 			podEvictorClient,
 			evictionPolicyGroupVersion,
@@ -285,6 +296,7 @@ func RunDeschedulerStrategies(ctx context.Context, rs *options.DeschedulerServer
 			deschedulerPolicy.MaxNoOfPodsToEvictPerNamespace,
 			nodes,
 			!rs.DisableMetrics,
+			deschedulerPolicy.MetricsConfig,
 		)
 
 		for name, strategy := range deschedulerPolicy.Strategies {