Refactor workqueue metrics

Signed-off-by: João Vilaça <jvilaca@redhat.com>

cmd/virt-api/BUILD.bazel

@@ -6,7 +6,6 @@ go_library(
     importpath = "kubevirt.io/kubevirt/cmd/virt-api",
     visibility = ["//visibility:private"],
     deps = [
-        "//pkg/monitoring/workqueue/prometheus:go_default_library",
         "//pkg/service:go_default_library",
         "//pkg/virt-api:go_default_library",
         "//staging/src/kubevirt.io/client-go/log:go_default_library",

cmd/virt-api/virt-api.go

@@ -22,7 +22,6 @@ package main
 import (
 	klog "kubevirt.io/client-go/log"
 
-	_ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics
 	"kubevirt.io/kubevirt/pkg/service"
 	virt_api "kubevirt.io/kubevirt/pkg/virt-api"
 )

cmd/virt-controller/BUILD.bazel

@@ -6,7 +6,6 @@ go_library(
     importpath = "kubevirt.io/kubevirt/cmd/virt-controller",
     visibility = ["//visibility:private"],
     deps = [
-        "//pkg/monitoring/workqueue/prometheus:go_default_library",
         "//pkg/virt-controller/watch:go_default_library",
     ],
 )

cmd/virt-controller/virt-controller.go

@@ -20,7 +20,6 @@
 package main
 
 import (
-	_ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics
 	"kubevirt.io/kubevirt/pkg/virt-controller/watch"
 )
 

cmd/virt-handler/BUILD.bazel

@@ -14,7 +14,6 @@ go_library(
         "//pkg/monitoring/domainstats/prometheus:go_default_library",
         "//pkg/monitoring/metrics/virt-handler:go_default_library",
         "//pkg/monitoring/profiler:go_default_library",
-        "//pkg/monitoring/workqueue/prometheus:go_default_library",
         "//pkg/safepath:go_default_library",
         "//pkg/service:go_default_library",
         "//pkg/util:go_default_library",

cmd/virt-handler/virt-handler.go

@@ -74,7 +74,6 @@ import (
 	promdomain "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus" // import for prometheus metrics
 	metrics "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler"
 	"kubevirt.io/kubevirt/pkg/monitoring/profiler"
-	_ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics
 	"kubevirt.io/kubevirt/pkg/service"
 	"kubevirt.io/kubevirt/pkg/util"
 	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"

cmd/virt-operator/BUILD.bazel

@@ -6,7 +6,6 @@ go_library(
     importpath = "kubevirt.io/kubevirt/cmd/virt-operator",
     visibility = ["//visibility:private"],
     deps = [
-        "//pkg/monitoring/workqueue/prometheus:go_default_library",
         "//pkg/virt-operator:go_default_library",
     ],
 )

cmd/virt-operator/virt-operator.go

@@ -20,7 +20,6 @@
 package main
 
 import (
-	_ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics
 	virt_operator "kubevirt.io/kubevirt/pkg/virt-operator"
 )
 

pkg/monitoring/metrics/common/workqueue/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["metrics.go"],
+    importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/k8s.io/client-go/util/workqueue:go_default_library",
+    ],
+)

pkg/monitoring/metrics/common/workqueue/metrics.go

@@ -0,0 +1,122 @@
+/*
+ * This file is part of the KubeVirt project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Copyright the KubeVirt Authors.
+ */
+
+package workqueue
+
+import (
+	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
+	"github.com/prometheus/client_golang/prometheus"
+	k8sworkqueue "k8s.io/client-go/util/workqueue"
+)
+
+func SetupMetrics() error {
+	k8sworkqueue.SetProvider(prometheusMetricsProvider{})
+	return nil
+}
+
+type prometheusMetricsProvider struct{}
+
+func (_ prometheusMetricsProvider) NewDepthMetric(name string) k8sworkqueue.GaugeMetric {
+	depth := operatormetrics.NewGauge(operatormetrics.MetricOpts{
+		Name:        "kubevirt_workqueue_depth",
+		Help:        "Current depth of workqueue",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	_ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{depth})
+
+	return depth
+}
+
+func (_ prometheusMetricsProvider) NewAddsMetric(name string) k8sworkqueue.CounterMetric {
+	adds := operatormetrics.NewCounter(operatormetrics.MetricOpts{
+		Name:        "kubevirt_workqueue_adds_total",
+		Help:        "Total number of adds handled by workqueue",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	_ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{adds})
+
+	return adds
+}
+
+func (_ prometheusMetricsProvider) NewLatencyMetric(name string) k8sworkqueue.HistogramMetric {
+	latency := operatormetrics.NewHistogram(
+		operatormetrics.MetricOpts{
+			Name:        "kubevirt_workqueue_queue_duration_seconds",
+			Help:        "How long an item stays in workqueue before being requested.",
+			ConstLabels: prometheus.Labels{"name": name},
+		},
+		prometheus.HistogramOpts{
+			Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
+		},
+	)
+	_ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{latency})
+
+	return latency
+}
+
+func (_ prometheusMetricsProvider) NewWorkDurationMetric(name string) k8sworkqueue.HistogramMetric {
+	workDuration := operatormetrics.NewHistogram(
+		operatormetrics.MetricOpts{
+			Name:        "kubevirt_workqueue_work_duration_seconds",
+			Help:        "How long in seconds processing an item from workqueue takes.",
+			ConstLabels: prometheus.Labels{"name": name},
+		},
+		prometheus.HistogramOpts{
+			Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10),
+		},
+	)
+	_ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{workDuration})
+
+	return workDuration
+}
+
+func (_ prometheusMetricsProvider) NewRetriesMetric(name string) k8sworkqueue.CounterMetric {
+	retries := operatormetrics.NewCounter(operatormetrics.MetricOpts{
+		Name:        "kubevirt_workqueue_retries_total",
+		Help:        "Total number of retries handled by workqueue",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	_ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{retries})
+
+	return retries
+}
+
+func (_ prometheusMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) k8sworkqueue.SettableGaugeMetric {
+	longestRunningProcessor := operatormetrics.NewGauge(operatormetrics.MetricOpts{
+		Name:        "kubevirt_workqueue_longest_running_processor_seconds",
+		Help:        "How many seconds has the longest running processor for workqueue been running.",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	_ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{longestRunningProcessor})
+
+	return longestRunningProcessor
+}
+
+func (_ prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) k8sworkqueue.SettableGaugeMetric {
+	unfinishedWork := operatormetrics.NewGauge(operatormetrics.MetricOpts{
+		Name: "kubevirt_workqueue_unfinished_work_seconds",
+		Help: "How many seconds of work has done that is in progress and hasn't " +
+			"been observed by work_duration. Large values indicate stuck " +
+			"threads. One can deduce the number of stuck threads by observing " +
+			"the rate at which this increases.",
+		ConstLabels: prometheus.Labels{"name": name},
+	})
+	_ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{unfinishedWork})
+
+	return unfinishedWork
+}

pkg/monitoring/metrics/virt-api/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/monitoring/metrics/common/client:go_default_library",
+        "//pkg/monitoring/metrics/common/workqueue:go_default_library",
         "//staging/src/kubevirt.io/api/core/v1:go_default_library",
         "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
     ],

pkg/monitoring/metrics/virt-api/metrics.go

@@ -23,13 +23,18 @@ import (
 	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
 
 	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client"
+	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue"
 )
 
 func SetupMetrics() error {
 	if err := client.SetupMetrics(); err != nil {
 		return err
 	}
 
+	if err := workqueue.SetupMetrics(); err != nil {
+		return err
+	}
+
 	return operatormetrics.RegisterMetrics(
 		connectionMetrics,
 		vmMetrics,

pkg/monitoring/metrics/virt-controller/BUILD.bazel

@@ -16,6 +16,7 @@ go_library(
     deps = [
         "//pkg/controller:go_default_library",
         "//pkg/monitoring/metrics/common/client:go_default_library",
+        "//pkg/monitoring/metrics/common/workqueue:go_default_library",
         "//pkg/util/migrations:go_default_library",
         "//pkg/virt-config:go_default_library",
         "//staging/src/kubevirt.io/api/core/v1:go_default_library",

pkg/monitoring/metrics/virt-controller/metrics.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 
 	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client"
+	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue"
 	virtconfig "kubevirt.io/kubevirt/pkg/virt-config"
 )
 
@@ -71,6 +72,10 @@ func SetupMetrics(
 		return err
 	}
 
+	if err := workqueue.SetupMetrics(); err != nil {
+		return err
+	}
+
 	if err := operatormetrics.RegisterMetrics(metrics...); err != nil {
 		return err
 	}

pkg/monitoring/metrics/virt-handler/BUILD.bazel

@@ -7,6 +7,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/monitoring/metrics/common/client:go_default_library",
+        "//pkg/monitoring/metrics/common/workqueue:go_default_library",
         "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
     ],
 )

pkg/monitoring/metrics/virt-handler/metrics.go

@@ -22,9 +22,14 @@ import (
 	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
 
 	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client"
+	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue"
 )
 
 func SetupMetrics() error {
+	if err := workqueue.SetupMetrics(); err != nil {
+		return err
+	}
+
 	return client.SetupMetrics()
 }
 

pkg/monitoring/metrics/virt-operator/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//pkg/monitoring/metrics/common/client:go_default_library",
+        "//pkg/monitoring/metrics/common/workqueue:go_default_library",
         "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library",
     ],
 )

pkg/monitoring/metrics/virt-operator/metrics.go

@@ -23,13 +23,18 @@ import (
 	"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
 
 	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client"
+	"kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue"
 )
 
 func SetupMetrics() error {
 	if err := client.SetupMetrics(); err != nil {
 		return err
 	}
 
+	if err := workqueue.SetupMetrics(); err != nil {
+		return err
+	}
+
 	return operatormetrics.RegisterMetrics(
 		configurationMetrics,
 		operatorMetrics,