From 58fb25a7f0b99a234fef25ec964a5e9e653e34b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Vila=C3=A7a?= Date: Fri, 8 Mar 2024 17:23:57 +0000 Subject: [PATCH] Refactor workqueue metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: João Vilaça --- cmd/virt-api/BUILD.bazel | 1 - cmd/virt-api/virt-api.go | 1 - cmd/virt-controller/BUILD.bazel | 1 - cmd/virt-controller/virt-controller.go | 1 - cmd/virt-handler/BUILD.bazel | 1 - cmd/virt-handler/virt-handler.go | 1 - cmd/virt-operator/BUILD.bazel | 1 - cmd/virt-operator/virt-operator.go | 1 - .../metrics/common/workqueue/BUILD.bazel | 13 ++ .../metrics/common/workqueue/metrics.go | 122 ++++++++++++++++++ pkg/monitoring/metrics/virt-api/BUILD.bazel | 1 + pkg/monitoring/metrics/virt-api/metrics.go | 5 + .../metrics/virt-controller/BUILD.bazel | 1 + .../metrics/virt-controller/metrics.go | 5 + .../metrics/virt-handler/BUILD.bazel | 1 + .../metrics/virt-handler/metrics.go | 5 + .../metrics/virt-operator/BUILD.bazel | 1 + .../metrics/virt-operator/metrics.go | 5 + 18 files changed, 159 insertions(+), 8 deletions(-) create mode 100644 pkg/monitoring/metrics/common/workqueue/BUILD.bazel create mode 100644 pkg/monitoring/metrics/common/workqueue/metrics.go diff --git a/cmd/virt-api/BUILD.bazel b/cmd/virt-api/BUILD.bazel index cc117078152c..d519484c4706 100644 --- a/cmd/virt-api/BUILD.bazel +++ b/cmd/virt-api/BUILD.bazel @@ -6,7 +6,6 @@ go_library( importpath = "kubevirt.io/kubevirt/cmd/virt-api", visibility = ["//visibility:private"], deps = [ - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/service:go_default_library", "//pkg/virt-api:go_default_library", "//staging/src/kubevirt.io/client-go/log:go_default_library", diff --git a/cmd/virt-api/virt-api.go b/cmd/virt-api/virt-api.go index 42681a22f149..a83332d81ba3 100644 --- a/cmd/virt-api/virt-api.go +++ b/cmd/virt-api/virt-api.go @@ -22,7 +22,6 @@ package main import ( klog "kubevirt.io/client-go/log" - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics "kubevirt.io/kubevirt/pkg/service" virt_api "kubevirt.io/kubevirt/pkg/virt-api" ) diff --git a/cmd/virt-controller/BUILD.bazel b/cmd/virt-controller/BUILD.bazel index 2c040348f647..fb326c5322ae 100644 --- a/cmd/virt-controller/BUILD.bazel +++ b/cmd/virt-controller/BUILD.bazel @@ -6,7 +6,6 @@ go_library( importpath = "kubevirt.io/kubevirt/cmd/virt-controller", visibility = ["//visibility:private"], deps = [ - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/virt-controller/watch:go_default_library", ], ) diff --git a/cmd/virt-controller/virt-controller.go b/cmd/virt-controller/virt-controller.go index 74a2c6ff3cf5..9d225846da75 100644 --- a/cmd/virt-controller/virt-controller.go +++ b/cmd/virt-controller/virt-controller.go @@ -20,7 +20,6 @@ package main import ( - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics "kubevirt.io/kubevirt/pkg/virt-controller/watch" ) diff --git a/cmd/virt-handler/BUILD.bazel b/cmd/virt-handler/BUILD.bazel index 67d6f44f2f94..cff502536586 100644 --- a/cmd/virt-handler/BUILD.bazel +++ b/cmd/virt-handler/BUILD.bazel @@ -14,7 +14,6 @@ go_library( "//pkg/monitoring/domainstats/prometheus:go_default_library", "//pkg/monitoring/metrics/virt-handler:go_default_library", "//pkg/monitoring/profiler:go_default_library", - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/safepath:go_default_library", "//pkg/service:go_default_library", "//pkg/util:go_default_library", diff --git a/cmd/virt-handler/virt-handler.go b/cmd/virt-handler/virt-handler.go index 94dc3a3b8fa3..1611ecb2a0c0 100644 --- a/cmd/virt-handler/virt-handler.go +++ b/cmd/virt-handler/virt-handler.go @@ -74,7 +74,6 @@ import ( promdomain "kubevirt.io/kubevirt/pkg/monitoring/domainstats/prometheus" // import for prometheus metrics metrics "kubevirt.io/kubevirt/pkg/monitoring/metrics/virt-handler" "kubevirt.io/kubevirt/pkg/monitoring/profiler" - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics "kubevirt.io/kubevirt/pkg/service" "kubevirt.io/kubevirt/pkg/util" virtconfig "kubevirt.io/kubevirt/pkg/virt-config" diff --git a/cmd/virt-operator/BUILD.bazel b/cmd/virt-operator/BUILD.bazel index 27c972030077..19a71abe5773 100644 --- a/cmd/virt-operator/BUILD.bazel +++ b/cmd/virt-operator/BUILD.bazel @@ -6,7 +6,6 @@ go_library( importpath = "kubevirt.io/kubevirt/cmd/virt-operator", visibility = ["//visibility:private"], deps = [ - "//pkg/monitoring/workqueue/prometheus:go_default_library", "//pkg/virt-operator:go_default_library", ], ) diff --git a/cmd/virt-operator/virt-operator.go b/cmd/virt-operator/virt-operator.go index 52bfbf1c3daa..c6462c785890 100644 --- a/cmd/virt-operator/virt-operator.go +++ b/cmd/virt-operator/virt-operator.go @@ -20,7 +20,6 @@ package main import ( - _ "kubevirt.io/kubevirt/pkg/monitoring/workqueue/prometheus" // import for prometheus metrics virt_operator "kubevirt.io/kubevirt/pkg/virt-operator" ) diff --git a/pkg/monitoring/metrics/common/workqueue/BUILD.bazel b/pkg/monitoring/metrics/common/workqueue/BUILD.bazel new file mode 100644 index 000000000000..f28a3130b9bd --- /dev/null +++ b/pkg/monitoring/metrics/common/workqueue/BUILD.bazel @@ -0,0 +1,13 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["metrics.go"], + importpath = "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue", + visibility = ["//visibility:public"], + deps = [ + "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", + "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library", + "//vendor/k8s.io/client-go/util/workqueue:go_default_library", + ], +) diff --git a/pkg/monitoring/metrics/common/workqueue/metrics.go b/pkg/monitoring/metrics/common/workqueue/metrics.go new file mode 100644 index 000000000000..01580f3640ad --- /dev/null +++ b/pkg/monitoring/metrics/common/workqueue/metrics.go @@ -0,0 +1,122 @@ +/* + * This file is part of the KubeVirt project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright the KubeVirt Authors. + */ + +package workqueue + +import ( + "github.com/machadovilaca/operator-observability/pkg/operatormetrics" + "github.com/prometheus/client_golang/prometheus" + k8sworkqueue "k8s.io/client-go/util/workqueue" +) + +func SetupMetrics() error { + k8sworkqueue.SetProvider(prometheusMetricsProvider{}) + return nil +} + +type prometheusMetricsProvider struct{} + +func (_ prometheusMetricsProvider) NewDepthMetric(name string) k8sworkqueue.GaugeMetric { + depth := operatormetrics.NewGauge(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_depth", + Help: "Current depth of workqueue", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{depth}) + + return depth +} + +func (_ prometheusMetricsProvider) NewAddsMetric(name string) k8sworkqueue.CounterMetric { + adds := operatormetrics.NewCounter(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_adds_total", + Help: "Total number of adds handled by workqueue", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{adds}) + + return adds +} + +func (_ prometheusMetricsProvider) NewLatencyMetric(name string) k8sworkqueue.HistogramMetric { + latency := operatormetrics.NewHistogram( + operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_queue_duration_seconds", + Help: "How long an item stays in workqueue before being requested.", + ConstLabels: prometheus.Labels{"name": name}, + }, + prometheus.HistogramOpts{ + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), + }, + ) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{latency}) + + return latency +} + +func (_ prometheusMetricsProvider) NewWorkDurationMetric(name string) k8sworkqueue.HistogramMetric { + workDuration := operatormetrics.NewHistogram( + operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_work_duration_seconds", + Help: "How long in seconds processing an item from workqueue takes.", + ConstLabels: prometheus.Labels{"name": name}, + }, + prometheus.HistogramOpts{ + Buckets: prometheus.ExponentialBuckets(10e-9, 10, 10), + }, + ) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{workDuration}) + + return workDuration +} + +func (_ prometheusMetricsProvider) NewRetriesMetric(name string) k8sworkqueue.CounterMetric { + retries := operatormetrics.NewCounter(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_retries_total", + Help: "Total number of retries handled by workqueue", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{retries}) + + return retries +} + +func (_ prometheusMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) k8sworkqueue.SettableGaugeMetric { + longestRunningProcessor := operatormetrics.NewGauge(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_longest_running_processor_seconds", + Help: "How many seconds has the longest running processor for workqueue been running.", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{longestRunningProcessor}) + + return longestRunningProcessor +} + +func (_ prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) k8sworkqueue.SettableGaugeMetric { + unfinishedWork := operatormetrics.NewGauge(operatormetrics.MetricOpts{ + Name: "kubevirt_workqueue_unfinished_work_seconds", + Help: "How many seconds of work has done that is in progress and hasn't " + + "been observed by work_duration. Large values indicate stuck " + + "threads. One can deduce the number of stuck threads by observing " + + "the rate at which this increases.", + ConstLabels: prometheus.Labels{"name": name}, + }) + _ = operatormetrics.RegisterMetrics([]operatormetrics.Metric{unfinishedWork}) + + return unfinishedWork +} diff --git a/pkg/monitoring/metrics/virt-api/BUILD.bazel b/pkg/monitoring/metrics/virt-api/BUILD.bazel index a70a86fa619d..5324455f8b60 100644 --- a/pkg/monitoring/metrics/virt-api/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-api/BUILD.bazel @@ -11,6 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", ], diff --git a/pkg/monitoring/metrics/virt-api/metrics.go b/pkg/monitoring/metrics/virt-api/metrics.go index 18115eac640d..348d0f257ba8 100644 --- a/pkg/monitoring/metrics/virt-api/metrics.go +++ b/pkg/monitoring/metrics/virt-api/metrics.go @@ -23,6 +23,7 @@ import ( "github.com/machadovilaca/operator-observability/pkg/operatormetrics" "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" ) func SetupMetrics() error { @@ -30,6 +31,10 @@ func SetupMetrics() error { return err } + if err := workqueue.SetupMetrics(); err != nil { + return err + } + return operatormetrics.RegisterMetrics( connectionMetrics, vmMetrics, diff --git a/pkg/monitoring/metrics/virt-controller/BUILD.bazel b/pkg/monitoring/metrics/virt-controller/BUILD.bazel index d580b88af986..46d50d155146 100644 --- a/pkg/monitoring/metrics/virt-controller/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-controller/BUILD.bazel @@ -16,6 +16,7 @@ go_library( deps = [ "//pkg/controller:go_default_library", "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", "//pkg/util/migrations:go_default_library", "//pkg/virt-config:go_default_library", "//staging/src/kubevirt.io/api/core/v1:go_default_library", diff --git a/pkg/monitoring/metrics/virt-controller/metrics.go b/pkg/monitoring/metrics/virt-controller/metrics.go index 061cf319f7bb..0c528518490c 100644 --- a/pkg/monitoring/metrics/virt-controller/metrics.go +++ b/pkg/monitoring/metrics/virt-controller/metrics.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/tools/cache" "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" virtconfig "kubevirt.io/kubevirt/pkg/virt-config" ) @@ -71,6 +72,10 @@ func SetupMetrics( return err } + if err := workqueue.SetupMetrics(); err != nil { + return err + } + if err := operatormetrics.RegisterMetrics(metrics...); err != nil { return err } diff --git a/pkg/monitoring/metrics/virt-handler/BUILD.bazel b/pkg/monitoring/metrics/virt-handler/BUILD.bazel index 95b0b449479b..8f0e3cd74c52 100644 --- a/pkg/monitoring/metrics/virt-handler/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-handler/BUILD.bazel @@ -7,6 +7,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", ], ) diff --git a/pkg/monitoring/metrics/virt-handler/metrics.go b/pkg/monitoring/metrics/virt-handler/metrics.go index 03b221a3266f..8ef0e023711e 100644 --- a/pkg/monitoring/metrics/virt-handler/metrics.go +++ b/pkg/monitoring/metrics/virt-handler/metrics.go @@ -22,9 +22,14 @@ import ( "github.com/machadovilaca/operator-observability/pkg/operatormetrics" "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" ) func SetupMetrics() error { + if err := workqueue.SetupMetrics(); err != nil { + return err + } + return client.SetupMetrics() } diff --git a/pkg/monitoring/metrics/virt-operator/BUILD.bazel b/pkg/monitoring/metrics/virt-operator/BUILD.bazel index 430bd690df7b..04b6c0fc2f4e 100644 --- a/pkg/monitoring/metrics/virt-operator/BUILD.bazel +++ b/pkg/monitoring/metrics/virt-operator/BUILD.bazel @@ -11,6 +11,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/monitoring/metrics/common/client:go_default_library", + "//pkg/monitoring/metrics/common/workqueue:go_default_library", "//vendor/github.com/machadovilaca/operator-observability/pkg/operatormetrics:go_default_library", ], ) diff --git a/pkg/monitoring/metrics/virt-operator/metrics.go b/pkg/monitoring/metrics/virt-operator/metrics.go index a111bb3af233..bb0c1a09ac06 100644 --- a/pkg/monitoring/metrics/virt-operator/metrics.go +++ b/pkg/monitoring/metrics/virt-operator/metrics.go @@ -23,6 +23,7 @@ import ( "github.com/machadovilaca/operator-observability/pkg/operatormetrics" "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/client" + "kubevirt.io/kubevirt/pkg/monitoring/metrics/common/workqueue" ) func SetupMetrics() error { @@ -30,6 +31,10 @@ func SetupMetrics() error { return err } + if err := workqueue.SetupMetrics(); err != nil { + return err + } + return operatormetrics.RegisterMetrics( configurationMetrics, operatorMetrics,