-
Notifications
You must be signed in to change notification settings - Fork 38.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Log prometheus metric registration error and fix CRD metric names #71767
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,7 @@ package prometheus | |
|
||
import ( | ||
"k8s.io/client-go/util/workqueue" | ||
"k8s.io/klog" | ||
|
||
"github.com/prometheus/client_golang/prometheus" | ||
) | ||
|
@@ -37,7 +38,9 @@ func (prometheusMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetr | |
Name: "depth", | ||
Help: "Current depth of workqueue: " + name, | ||
}) | ||
prometheus.Register(depth) | ||
if err := prometheus.Register(depth); err != nil { | ||
klog.Errorf("failed to register depth metric %v: %v", name, err) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. shouldn't this me a more fatal error actually? It's a programming error. Those names are static strings, i.e. they don't come from the user. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. there are more failures caused by duplication that I didn't dig into (see the gist). We can have a followup issue to clean those up There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. it's not a complete list. kcm/ccm may also have such failure There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. created #73551 |
||
} | ||
return depth | ||
} | ||
|
||
|
@@ -47,7 +50,9 @@ func (prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMet | |
Name: "adds", | ||
Help: "Total number of adds handled by workqueue: " + name, | ||
}) | ||
prometheus.Register(adds) | ||
if err := prometheus.Register(adds); err != nil { | ||
klog.Errorf("failed to register adds metric %v: %v", name, err) | ||
} | ||
return adds | ||
} | ||
|
||
|
@@ -57,7 +62,9 @@ func (prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.Summary | |
Name: "queue_latency", | ||
Help: "How long an item stays in workqueue" + name + " before being requested.", | ||
}) | ||
prometheus.Register(latency) | ||
if err := prometheus.Register(latency); err != nil { | ||
klog.Errorf("failed to register latency metric %v: %v", name, err) | ||
} | ||
return latency | ||
} | ||
|
||
|
@@ -67,7 +74,9 @@ func (prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.Su | |
Name: "work_duration", | ||
Help: "How long processing an item from workqueue" + name + " takes.", | ||
}) | ||
prometheus.Register(workDuration) | ||
if err := prometheus.Register(workDuration); err != nil { | ||
klog.Errorf("failed to register work_duration metric %v: %v", name, err) | ||
} | ||
return workDuration | ||
} | ||
|
||
|
@@ -80,7 +89,9 @@ func (prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) wor | |
"values indicate stuck threads. One can deduce the number of stuck " + | ||
"threads by observing the rate at which this increases.", | ||
}) | ||
prometheus.Register(unfinished) | ||
if err := prometheus.Register(unfinished); err != nil { | ||
klog.Errorf("failed to register unfinished_work_seconds metric %v: %v", name, err) | ||
} | ||
return unfinished | ||
} | ||
|
||
|
@@ -91,7 +102,9 @@ func (prometheusMetricsProvider) NewLongestRunningProcessorMicrosecondsMetric(na | |
Help: "How many microseconds has the longest running " + | ||
"processor for " + name + " been running.", | ||
}) | ||
prometheus.Register(unfinished) | ||
if err := prometheus.Register(unfinished); err != nil { | ||
klog.Errorf("failed to register longest_running_processor_microseconds metric %v: %v", name, err) | ||
} | ||
return unfinished | ||
} | ||
|
||
|
@@ -101,6 +114,8 @@ func (prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.Counter | |
Name: "retries", | ||
Help: "Total number of retries handled by workqueue: " + name, | ||
}) | ||
prometheus.Register(retries) | ||
if err := prometheus.Register(retries); err != nil { | ||
klog.Errorf("failed to register retries metric %v: %v", name, err) | ||
} | ||
return retries | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -79,7 +79,7 @@ func NewCRDFinalizer( | |
crdLister: crdInformer.Lister(), | ||
crdSynced: crdInformer.Informer().HasSynced, | ||
crClientGetter: crClientGetter, | ||
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "CustomResourceDefinition-CRDFinalizer"), | ||
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_finalizer"), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. crd_finalizer_controller? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. crd_finalizer_controller sounds like a controller for crd finalizer to me also we tend to omit "controller" from the name for other in-tree controllers: https://github.com/kubernetes/kubernetes/search?q=NewNamedRateLimitingQueue&unscoped_q=NewNamedRateLimitingQueue |
||
} | ||
|
||
crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
to understand: dash was never allowed and this metric was never registered?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nvmd. It's the queue name.