From 6fa8e3dbeb055efed73b715fc5d895f99f6250db Mon Sep 17 00:00:00 2001 From: Michael Tsang Date: Thu, 20 Jun 2019 11:56:58 +0100 Subject: [PATCH 1/5] Add prometheus metric for certificate ready condition Signed-off-by: Michael Tsang Co-authored-by: Aleksandr Seleznev --- pkg/metrics/metrics.go | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 45a981364a6..4ab7f7ba07b 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -17,6 +17,7 @@ limitations under the License. // Package metrics contains global structures related to metrics collection // cert-manager exposes the following metrics: // certificate_expiration_timestamp_seconds{name, namespace} +// certificate_ready_status{name, namespace, condition} package metrics import ( @@ -52,6 +53,8 @@ const ( prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB ) +var readyConditionStatuses = [...]v1alpha1.ConditionStatus{v1alpha1.ConditionTrue, v1alpha1.ConditionFalse, v1alpha1.ConditionUnknown} + // Default set of metrics var Default = New(logf.NewContext(context.Background(), logf.Log.WithName("metrics"))) @@ -64,6 +67,15 @@ var CertificateExpiryTimeSeconds = prometheus.NewGaugeVec( []string{"name", "namespace"}, ) +var CertificateReadyStatus = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: "certificate_ready_status", + Help: "The ready status of the Certificate.", + }, + []string{"name", "namespace", "condition"}, +) + // ACMEClientRequestCount is a Prometheus summary to collect the number of // requests made to each endpoint with the ACME client. var ACMEClientRequestCount = prometheus.NewCounterVec( @@ -117,6 +129,7 @@ type Metrics struct { // TODO (@dippynark): switch this to use an interface to make it testable registry *prometheus.Registry CertificateExpiryTimeSeconds *prometheus.GaugeVec + CertificateReadyStatus *prometheus.GaugeVec ACMEClientRequestDurationSeconds *prometheus.SummaryVec ACMEClientRequestCount *prometheus.CounterVec ControllerSyncCallCount *prometheus.CounterVec @@ -138,6 +151,7 @@ func New(ctx context.Context) *Metrics { activeCertificates: nil, registry: prometheus.NewRegistry(), CertificateExpiryTimeSeconds: CertificateExpiryTimeSeconds, + CertificateReadyStatus: CertificateReadyStatus, ACMEClientRequestDurationSeconds: ACMEClientRequestDurationSeconds, ACMEClientRequestCount: ACMEClientRequestCount, ControllerSyncCallCount: ControllerSyncCallCount, @@ -168,6 +182,7 @@ func (m *Metrics) Start(stopCh <-chan struct{}) { log := logf.FromContext(m.ctx) m.registry.MustRegister(m.CertificateExpiryTimeSeconds) + m.registry.MustRegister(m.CertificateReadyStatus) m.registry.MustRegister(m.ACMEClientRequestDurationSeconds) m.registry.MustRegister(m.ACMEClientRequestCount) m.registry.MustRegister(m.ControllerSyncCallCount) @@ -228,6 +243,42 @@ func (m *Metrics) SetActiveCertificates(cl cmlisters.CertificateLister) { m.activeCertificates = cl } +func (m *Metrics) UpdateCertificateStatus(crt *v1alpha1.Certificate) { + log := logf.FromContext(m.ctx) + log = logf.WithResource(log, crt) + + log.V(logf.DebugLevel).Info("attempting to retrieve ready status for certificate") + for _, c := range crt.Status.Conditions { + switch c.Type { + case v1alpha1.CertificateConditionReady: + updateCertificateReadyStatus(crt, c.Status) + } + } +} + +func updateCertificateReadyStatus(crt *v1alpha1.Certificate, current v1alpha1.ConditionStatus) { + + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt) + if err != nil { + return + } + + registeredCertificates.mtx.Lock() + defer registeredCertificates.mtx.Unlock() + for _, condition := range readyConditionStatuses { + value := 0.0 + if current == condition { + value = 1.0 + } + CertificateReadyStatus.With(prometheus.Labels{ + "name": crt.Name, + "namespace": crt.Namespace, + "condition": string(condition), + }).Set(value) + } + registeredCertificates.certificates[key] = struct{}{} +} + func (m *Metrics) cleanUp() { log := logf.FromContext(m.ctx) log.V(logf.DebugLevel).Info("attempting to clean up metrics for recently deleted certificates") @@ -276,6 +327,14 @@ func cleanUpCertificates(activeCrts []*v1alpha1.Certificate) { "name": name, "namespace": namespace, }) + + for _, condition := range readyConditionStatuses { + CertificateReadyStatus.Delete(prometheus.Labels{ + "name": name, + "namespace": namespace, + "condition": string(condition), + }) + } delete(registeredCertificates.certificates, key) } } From 19eeb210c7306674dfd1499bdf8ae096c54bf6ef Mon Sep 17 00:00:00 2001 From: Michael Tsang Date: Thu, 20 Jun 2019 14:39:32 +0100 Subject: [PATCH 2/5] Add tests for updating certificate status Signed-off-by: Michael Tsang Co-authored-by: Aleksandr Seleznev --- pkg/metrics/metrics.go | 3 +- pkg/metrics/metrics_test.go | 178 ++++++++++++++++++++++++++---------- 2 files changed, 129 insertions(+), 52 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 4ab7f7ba07b..0eece338437 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -71,7 +71,7 @@ var CertificateReadyStatus = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: namespace, Name: "certificate_ready_status", - Help: "The ready status of the Certificate.", + Help: "The ready status of the certificate.", }, []string{"name", "namespace", "condition"}, ) @@ -257,7 +257,6 @@ func (m *Metrics) UpdateCertificateStatus(crt *v1alpha1.Certificate) { } func updateCertificateReadyStatus(crt *v1alpha1.Certificate, current v1alpha1.ConditionStatus) { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(crt) if err != nil { return diff --git a/pkg/metrics/metrics_test.go b/pkg/metrics/metrics_test.go index 65a6f1f4fcc..a12f044df96 100644 --- a/pkg/metrics/metrics_test.go +++ b/pkg/metrics/metrics_test.go @@ -28,6 +28,33 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func getReadyConditionStatus(crt *v1alpha1.Certificate) v1alpha1.ConditionStatus { + for _, c := range crt.Status.Conditions { + switch c.Type { + case v1alpha1.CertificateConditionReady: + return c.Status + } + } + return v1alpha1.ConditionUnknown +} + +func buildCertificate(name, namespace string, condition v1alpha1.ConditionStatus) *v1alpha1.Certificate { + return &v1alpha1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Status: v1alpha1.CertificateStatus{ + Conditions: []v1alpha1.CertificateCondition{ + { + Type: v1alpha1.CertificateConditionReady, + Status: condition, + }, + }, + }, + } +} + func TestUpdateCertificateExpiry(t *testing.T) { const metadata = ` # HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time. @@ -72,94 +99,135 @@ func TestUpdateCertificateExpiry(t *testing.T) { } } -func TestCleanUp(t *testing.T) { +func TestUpdateCertificateReadyStatus(t *testing.T) { const metadata = ` + # HELP certmanager_certificate_ready_status The ready status of the certificate. + # TYPE certmanager_certificate_ready_status gauge +` + + type testT struct { + crt *v1alpha1.Certificate + expected string + } + tests := map[string]testT{ + "ready status true is updated correctly": { + crt: buildCertificate("something", "default", v1alpha1.ConditionTrue), + expected: ` + certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0 + certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 1 + certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0 +`, + }, + "ready status false is updated correctly": { + crt: buildCertificate("something", "default", v1alpha1.ConditionFalse), + expected: ` + certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 1 + certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 0 +`, + }, + "ready status unknown is updated correctly": { + crt: buildCertificate("something", "default", v1alpha1.ConditionUnknown), + expected: ` + certmanager_certificate_ready_status{condition="False",name="something",namespace="default"} 0 + certmanager_certificate_ready_status{condition="True",name="something",namespace="default"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="something",namespace="default"} 1 +`, + }, + } + for n, test := range tests { + t.Run(n, func(t *testing.T) { + updateCertificateReadyStatus(test.crt, getReadyConditionStatus(test.crt)) + + if err := testutil.CollectAndCompare( + CertificateReadyStatus, + strings.NewReader(metadata+test.expected), + "certmanager_certificate_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } + }) + } +} + +func TestCleanUp(t *testing.T) { + const metadataExpiry = ` # HELP certmanager_certificate_expiration_timestamp_seconds The date after which the certificate expires. Expressed as a Unix Epoch Time. # TYPE certmanager_certificate_expiration_timestamp_seconds gauge ` + + const metadataReady = ` + # HELP certmanager_certificate_ready_status The ready status of the certificate. + # TYPE certmanager_certificate_ready_status gauge +` type testT struct { - active map[*v1alpha1.Certificate]*x509.Certificate - inactive map[*v1alpha1.Certificate]*x509.Certificate - expected string + active map[*v1alpha1.Certificate]*x509.Certificate + inactive map[*v1alpha1.Certificate]*x509.Certificate + expectedExpiry string + expectedReady string } tests := map[string]testT{ - "active and inactive": { + "inactive certificate metrics cleaned up while active certificate metrics kept": { active: map[*v1alpha1.Certificate]*x509.Certificate{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "something", - Namespace: "default", - }, - }: { + buildCertificate("active", "default", v1alpha1.ConditionTrue): { // fixed expiry time for testing NotAfter: time.Unix(2208988804, 0), }, }, inactive: map[*v1alpha1.Certificate]*x509.Certificate{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "something-else", - Namespace: "default", - }, - }: { + buildCertificate("inactive", "default", v1alpha1.ConditionTrue): { // fixed expiry time for testing NotAfter: time.Unix(2208988804, 0), }, }, - expected: ` - certmanager_certificate_expiration_timestamp_seconds{name="something",namespace="default"} 2.208988804e+09 + expectedExpiry: ` + certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09 +`, + expectedReady: ` + certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0 + certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1 + certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0 `, }, - "only active": { + "no metrics cleaned up when only active certificate metrics": { active: map[*v1alpha1.Certificate]*x509.Certificate{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "something", - Namespace: "default", - }, - }: { + buildCertificate("active", "default", v1alpha1.ConditionTrue): { // fixed expiry time for testing NotAfter: time.Unix(2208988804, 0), }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "something-else", - Namespace: "default", - }, - }: { + buildCertificate("also-active", "default", v1alpha1.ConditionTrue): { // fixed expiry time for testing NotAfter: time.Unix(2208988804, 0), }, }, inactive: map[*v1alpha1.Certificate]*x509.Certificate{}, - expected: ` - certmanager_certificate_expiration_timestamp_seconds{name="something",namespace="default"} 2.208988804e+09 - certmanager_certificate_expiration_timestamp_seconds{name="something-else",namespace="default"} 2.208988804e+09 + expectedExpiry: ` + certmanager_certificate_expiration_timestamp_seconds{name="active",namespace="default"} 2.208988804e+09 + certmanager_certificate_expiration_timestamp_seconds{name="also-active",namespace="default"} 2.208988804e+09 +`, + expectedReady: ` + certmanager_certificate_ready_status{condition="False",name="active",namespace="default"} 0 + certmanager_certificate_ready_status{condition="False",name="also-active",namespace="default"} 0 + certmanager_certificate_ready_status{condition="True",name="active",namespace="default"} 1 + certmanager_certificate_ready_status{condition="True",name="also-active",namespace="default"} 1 + certmanager_certificate_ready_status{condition="Unknown",name="active",namespace="default"} 0 + certmanager_certificate_ready_status{condition="Unknown",name="also-active",namespace="default"} 0 `, }, - "only inactive": { + "all metrics cleaned up when only inactive certificate metrics": { active: map[*v1alpha1.Certificate]*x509.Certificate{}, inactive: map[*v1alpha1.Certificate]*x509.Certificate{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "something", - Namespace: "default", - }, - }: { + buildCertificate("inactive", "default", v1alpha1.ConditionTrue): { // fixed expiry time for testing NotAfter: time.Unix(2208988804, 0), }, - { - ObjectMeta: metav1.ObjectMeta{ - Name: "something-else", - Namespace: "default", - }, - }: { + buildCertificate("also-inactive", "default", v1alpha1.ConditionTrue): { // fixed expiry time for testing NotAfter: time.Unix(2208988804, 0), }, }, - expected: "", + expectedExpiry: "", + expectedReady: "", }, } for n, test := range tests { @@ -169,9 +237,11 @@ func TestCleanUp(t *testing.T) { var activeCrts []*v1alpha1.Certificate for crt, cert := range test.active { updateX509Expiry(crt, cert) + updateCertificateReadyStatus(crt, getReadyConditionStatus(crt)) activeCrts = append(activeCrts, crt) } for crt, cert := range test.inactive { + updateCertificateReadyStatus(crt, getReadyConditionStatus(crt)) updateX509Expiry(crt, cert) } @@ -179,11 +249,19 @@ func TestCleanUp(t *testing.T) { if err := testutil.CollectAndCompare( CertificateExpiryTimeSeconds, - strings.NewReader(metadata+test.expected), + strings.NewReader(metadataExpiry+test.expectedExpiry), "certmanager_certificate_expiration_timestamp_seconds", ); err != nil { t.Errorf("unexpected collecting result:\n%s", err) } + + if err := testutil.CollectAndCompare( + CertificateReadyStatus, + strings.NewReader(metadataReady+test.expectedReady), + "certmanager_certificate_ready_status", + ); err != nil { + t.Errorf("unexpected collecting result:\n%s", err) + } }) } } From f3baab085ea2d5e3ab5fae7ca386273fdffd6f62 Mon Sep 17 00:00:00 2001 From: Michael Tsang Date: Thu, 20 Jun 2019 15:06:16 +0100 Subject: [PATCH 3/5] Add UpdateCertificateStatus call, comment implementation Signed-off-by: Michael Tsang --- pkg/controller/certificates/sync.go | 2 ++ pkg/metrics/metrics.go | 55 +++++++++++++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/pkg/controller/certificates/sync.go b/pkg/controller/certificates/sync.go index 6fd8e453ff6..3f90635870d 100644 --- a/pkg/controller/certificates/sync.go +++ b/pkg/controller/certificates/sync.go @@ -585,6 +585,8 @@ func generateLocallySignedTemporaryCertificate(crt *v1alpha1.Certificate, pk []b } func (c *controller) updateCertificateStatus(ctx context.Context, old, new *v1alpha1.Certificate) (*v1alpha1.Certificate, error) { + defer c.metrics.UpdateCertificateStatus(new) + log := logf.FromContext(ctx, "updateStatus") oldBytes, _ := json.Marshal(old.Status) newBytes, _ := json.Marshal(new.Status) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 0eece338437..e46caf99a53 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -236,11 +236,7 @@ func updateX509Expiry(crt *v1alpha1.Certificate, cert *x509.Certificate) { CertificateExpiryTimeSeconds.With(prometheus.Labels{ "name": crt.Name, "namespace": crt.Namespace}).Set(float64(expiryTime.Unix())) - registeredCertificates.certificates[key] = struct{}{} -} - -func (m *Metrics) SetActiveCertificates(cl cmlisters.CertificateLister) { - m.activeCertificates = cl + registerCertificateKey(key) } func (m *Metrics) UpdateCertificateStatus(crt *v1alpha1.Certificate) { @@ -275,9 +271,23 @@ func updateCertificateReadyStatus(crt *v1alpha1.Certificate, current v1alpha1.Co "condition": string(condition), }).Set(value) } + registerCertificateKey(key) +} + +// registerCertificateKey adds an entry in registeredCertificates to track +// which certificates have metrics stored in prometheus, allowing for easier +// clean-up. +// You MUST lock the mutex before calling this function, this ensures no other +// function is cleaning up while we are registering a certificate +func registerCertificateKey(key string) { registeredCertificates.certificates[key] = struct{}{} } +func (m *Metrics) SetActiveCertificates(cl cmlisters.CertificateLister) { + m.activeCertificates = cl +} + +// cleanUp removes any metrics which reference resources which no longer exist func (m *Metrics) cleanUp() { log := logf.FromContext(m.ctx) log.V(logf.DebugLevel).Info("attempting to clean up metrics for recently deleted certificates") @@ -296,6 +306,7 @@ func (m *Metrics) cleanUp() { cleanUpCertificates(activeCrts) } +// cleanUpCertificates removes metrics for recently deleted certificates func cleanUpCertificates(activeCrts []*v1alpha1.Certificate) { activeMap := make(map[string]struct{}, len(activeCrts)) for _, crt := range activeCrts { @@ -317,25 +328,31 @@ func cleanUpCertificates(activeCrts []*v1alpha1.Certificate) { } for _, key := range toCleanUp { - namespace, name, err := cache.SplitMetaNamespaceKey(key) - if err != nil { - continue - } + cleanUpCertificateByKey(key) + } +} + +// cleanUpCertificateByKey removes metrics which refer to a certificate, +// given the key of the certificate. +func cleanUpCertificateByKey(key string) { + namespace, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + return + } + + CertificateExpiryTimeSeconds.Delete(prometheus.Labels{ + "name": name, + "namespace": namespace, + }) - CertificateExpiryTimeSeconds.Delete(prometheus.Labels{ + for _, condition := range readyConditionStatuses { + CertificateReadyStatus.Delete(prometheus.Labels{ "name": name, "namespace": namespace, + "condition": string(condition), }) - - for _, condition := range readyConditionStatuses { - CertificateReadyStatus.Delete(prometheus.Labels{ - "name": name, - "namespace": namespace, - "condition": string(condition), - }) - } - delete(registeredCertificates.certificates, key) } + delete(registeredCertificates.certificates, key) } func (m *Metrics) IncrementSyncCallCount(controllerName string) { From 1317b8df621f111b3ae6f3914d9840285f2cb122 Mon Sep 17 00:00:00 2001 From: Michael Tsang Date: Fri, 21 Jun 2019 12:11:04 +0100 Subject: [PATCH 4/5] Add cleanup function slice Signed-off-by: Michael Tsang --- pkg/metrics/metrics.go | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index e46caf99a53..5ffda595c13 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -53,7 +53,7 @@ const ( prometheusMetricsServerMaxHeaderBytes = 1 << 20 // 1 MiB ) -var readyConditionStatuses = [...]v1alpha1.ConditionStatus{v1alpha1.ConditionTrue, v1alpha1.ConditionFalse, v1alpha1.ConditionUnknown} +var readyConditionStatuses = [...]string{string(v1alpha1.ConditionTrue), string(v1alpha1.ConditionFalse), string(v1alpha1.ConditionUnknown)} // Default set of metrics var Default = New(logf.NewContext(context.Background(), logf.Log.WithName("metrics"))) @@ -121,6 +121,15 @@ var registeredCertificates = &struct { var activeCertificates cmlisters.CertificateLister +var cleanUpFunctions = []func(string, string){ + metricCleanUpCertificate(CertificateExpiryTimeSeconds), + metricCleanUpCertificateWith(CertificateReadyStatus, readyConditionStatuses[:]), +} + +type cleanableMetric interface { + DeleteLabelValues(...string) bool +} + type Metrics struct { ctx context.Context http.Server @@ -262,7 +271,7 @@ func updateCertificateReadyStatus(crt *v1alpha1.Certificate, current v1alpha1.Co defer registeredCertificates.mtx.Unlock() for _, condition := range readyConditionStatuses { value := 0.0 - if current == condition { + if string(current) == condition { value = 1.0 } CertificateReadyStatus.With(prometheus.Labels{ @@ -332,6 +341,20 @@ func cleanUpCertificates(activeCrts []*v1alpha1.Certificate) { } } +func metricCleanUpCertificate(c cleanableMetric) func(string, string) { + return func(name, namespace string) { + c.DeleteLabelValues(name, namespace) + } +} + +func metricCleanUpCertificateWith(c cleanableMetric, additionalLabels []string) func(string, string) { + return func(name, namespace string) { + for _, label := range additionalLabels { + c.DeleteLabelValues(name, namespace, label) + } + } +} + // cleanUpCertificateByKey removes metrics which refer to a certificate, // given the key of the certificate. func cleanUpCertificateByKey(key string) { @@ -340,18 +363,10 @@ func cleanUpCertificateByKey(key string) { return } - CertificateExpiryTimeSeconds.Delete(prometheus.Labels{ - "name": name, - "namespace": namespace, - }) - - for _, condition := range readyConditionStatuses { - CertificateReadyStatus.Delete(prometheus.Labels{ - "name": name, - "namespace": namespace, - "condition": string(condition), - }) + for _, f := range cleanUpFunctions { + f(name, namespace) } + delete(registeredCertificates.certificates, key) } From c2fd63a7a66c32ac91ee0fd7bf622519bbfb479b Mon Sep 17 00:00:00 2001 From: Michael Tsang Date: Fri, 21 Jun 2019 14:34:47 +0100 Subject: [PATCH 5/5] Add comments to refactored metric clean up Signed-off-by: Michael Tsang --- pkg/metrics/metrics.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 5ffda595c13..c625e8a3aab 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -121,6 +121,8 @@ var registeredCertificates = &struct { var activeCertificates cmlisters.CertificateLister +// cleanUpFunctions are functions called to clean up metrics which refer to +// deleted certificates, inputs are name and namespace of the certificate var cleanUpFunctions = []func(string, string){ metricCleanUpCertificate(CertificateExpiryTimeSeconds), metricCleanUpCertificateWith(CertificateReadyStatus, readyConditionStatuses[:]), @@ -208,6 +210,7 @@ func (m *Metrics) Start(stopCh <-chan struct{}) { }() + // clean up metrics referring to deleted resources every minute go wait.Until(func() { m.cleanUp() }, time.Minute, stopCh) m.waitShutdown(stopCh) @@ -341,12 +344,19 @@ func cleanUpCertificates(activeCrts []*v1alpha1.Certificate) { } } +// metricCleanUpCertificate creates a clean up function which deletes the entry +// (if any) for a certificate in the given metric func metricCleanUpCertificate(c cleanableMetric) func(string, string) { return func(name, namespace string) { c.DeleteLabelValues(name, namespace) } } +// metricCleanUpCertificateWith creates a clean up function which deletes the +// entries (if any) for a certificate in the given metric, iterating over the +// additional labels. +// This is used if the metric keys on data in addition to the name and +// namespace. func metricCleanUpCertificateWith(c cleanableMetric, additionalLabels []string) func(string, string) { return func(name, namespace string) { for _, label := range additionalLabels { @@ -363,6 +373,7 @@ func cleanUpCertificateByKey(key string) { return } + // apply all the clean up functions for _, f := range cleanUpFunctions { f(name, namespace) }