diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD index 21a8fc02719d..a34513df7f59 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/BUILD @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["exec.go"], + srcs = [ + "exec.go", + "metrics.go", + ], importmap = "k8s.io/kubernetes/vendor/k8s.io/client-go/plugin/pkg/client/auth/exec", importpath = "k8s.io/client-go/plugin/pkg/client/auth/exec", visibility = ["//visibility:public"], @@ -16,6 +19,7 @@ go_library( "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1:go_default_library", "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication/v1beta1:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", + "//staging/src/k8s.io/client-go/tools/metrics:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", "//staging/src/k8s.io/client-go/util/connrotation:go_default_library", "//vendor/github.com/davecgh/go-spew/spew:go_default_library", @@ -26,7 +30,10 @@ go_library( go_test( name = "go_default_test", - srcs = ["exec_test.go"], + srcs = [ + "exec_test.go", + "metrics_test.go", + ], data = glob(["testdata/**"]), embed = [":go_default_library"], deps = [ @@ -34,6 +41,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", "//staging/src/k8s.io/client-go/pkg/apis/clientauthentication:go_default_library", "//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library", + "//staging/src/k8s.io/client-go/tools/metrics:go_default_library", "//staging/src/k8s.io/client-go/transport:go_default_library", ], ) diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go index 741729bb5d60..9f029ee0a972 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "crypto/tls" + "crypto/x509" "errors" "fmt" "io" @@ -42,6 +43,7 @@ import ( "k8s.io/client-go/pkg/apis/clientauthentication/v1alpha1" "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/tools/metrics" "k8s.io/client-go/transport" "k8s.io/client-go/util/connrotation" "k8s.io/klog" @@ -260,6 +262,8 @@ func (a *Authenticator) cert() (*tls.Certificate, error) { func (a *Authenticator) getCreds() (*credentials, error) { a.mu.Lock() defer a.mu.Unlock() + defer expirationMetrics.report(time.Now) + if a.cachedCreds != nil && !a.credsExpired() { return a.cachedCreds, nil } @@ -267,6 +271,7 @@ func (a *Authenticator) getCreds() (*credentials, error) { if err := a.refreshCredsLocked(nil); err != nil { return nil, err } + return a.cachedCreds, nil } @@ -355,6 +360,17 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err if err != nil { return fmt.Errorf("failed parsing client key/certificate: %v", err) } + + // Leaf is initialized to be nil: + // https://golang.org/pkg/crypto/tls/#X509KeyPair + // Leaf certificate is the first certificate: + // https://golang.org/pkg/crypto/tls/#Certificate + // Populating leaf is useful for quickly accessing the underlying x509 + // certificate values. + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return fmt.Errorf("failed parsing client leaf certificate: %v", err) + } newCreds.cert = &cert } @@ -362,10 +378,20 @@ func (a *Authenticator) refreshCredsLocked(r *clientauthentication.Response) err a.cachedCreds = newCreds // Only close all connections when TLS cert rotates. Token rotation doesn't // need the extra noise. - if len(a.onRotateList) > 0 && oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) { + if oldCreds != nil && !reflect.DeepEqual(oldCreds.cert, a.cachedCreds.cert) { + // Can be nil if the exec auth plugin only returned token auth. + if oldCreds.cert != nil && oldCreds.cert.Leaf != nil { + metrics.ClientCertRotationAge.Observe(time.Now().Sub(oldCreds.cert.Leaf.NotBefore)) + } for _, onRotate := range a.onRotateList { onRotate() } } + + expiry := time.Time{} + if a.cachedCreds.cert != nil && a.cachedCreds.cert.Leaf != nil { + expiry = a.cachedCreds.cert.Leaf.NotAfter + } + expirationMetrics.set(a, expiry) return nil } diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec_test.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec_test.go index d8f94cc099b3..9bef7090fb7e 100644 --- a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec_test.go +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/exec_test.go @@ -97,6 +97,10 @@ func init() { if err != nil { panic(err) } + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + panic(err) + } validCert = &cert } @@ -760,7 +764,7 @@ func TestConcurrentUpdateTransportConfig(t *testing.T) { } // genClientCert generates an x509 certificate for testing. Certificate and key -// are returned in PEM encoding. +// are returned in PEM encoding. The generated cert expires in 24 hours. func genClientCert(t *testing.T) ([]byte, []byte) { key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go new file mode 100644 index 000000000000..6ec6556c073e --- /dev/null +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics.go @@ -0,0 +1,64 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exec + +import ( + "sync" + "time" + + "k8s.io/client-go/tools/metrics" +) + +type certificateExpirationTracker struct { + mu sync.RWMutex + m map[*Authenticator]time.Time + earliest time.Time +} + +var expirationMetrics = &certificateExpirationTracker{m: map[*Authenticator]time.Time{}} + +// set stores the given expiration time and updates the updates earliest. +func (c *certificateExpirationTracker) set(a *Authenticator, t time.Time) { + c.mu.Lock() + defer c.mu.Unlock() + c.m[a] = t + + // update earliest + earliest := time.Time{} + for _, t := range c.m { + if t.IsZero() { + continue + } + if earliest.IsZero() || earliest.After(t) { + earliest = t + } + } + c.earliest = earliest +} + +// report reports the ttl to the earliest reported expiration time. +// If no Authenticators have reported a certificate expiration, this reports nil. +func (c *certificateExpirationTracker) report(now func() time.Time) { + c.mu.RLock() + defer c.mu.RUnlock() + if c.earliest.IsZero() { + metrics.ClientCertTTL.Set(nil) + } else { + ttl := c.earliest.Sub(now()) + metrics.ClientCertTTL.Set(&ttl) + } +} diff --git a/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go new file mode 100644 index 000000000000..974f433c8562 --- /dev/null +++ b/staging/src/k8s.io/client-go/plugin/pkg/client/auth/exec/metrics_test.go @@ -0,0 +1,106 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package exec + +import ( + "testing" + "time" + + "k8s.io/client-go/tools/metrics" +) + +type mockTTLGauge struct { + v *time.Duration +} + +func (m *mockTTLGauge) Set(d *time.Duration) { + m.v = d +} + +func ptr(d time.Duration) *time.Duration { + return &d +} + +func TestCertificateExpirationTracker(t *testing.T) { + now := time.Now() + nowFn := func() time.Time { return now } + mockMetric := &mockTTLGauge{} + realMetric := metrics.ClientCertTTL + metrics.ClientCertTTL = mockMetric + defer func() { + metrics.ClientCertTTL = realMetric + }() + + tracker := &certificateExpirationTracker{m: map[*Authenticator]time.Time{}} + tracker.report(nowFn) + if mockMetric.v != nil { + t.Error("empty tracker should record nil value") + } + + firstAuthenticator := &Authenticator{} + secondAuthenticator := &Authenticator{} + for _, tc := range []struct { + desc string + auth *Authenticator + time time.Time + want *time.Duration + }{ + { + desc: "ttl for one authenticator", + auth: firstAuthenticator, + time: now.Add(time.Minute * 10), + want: ptr(time.Minute * 10), + }, + { + desc: "second authenticator shorter ttl", + auth: secondAuthenticator, + time: now.Add(time.Minute * 5), + want: ptr(time.Minute * 5), + }, + { + desc: "update shorter to be longer", + auth: secondAuthenticator, + time: now.Add(time.Minute * 15), + want: ptr(time.Minute * 10), + }, + { + desc: "update shorter to be zero time", + auth: firstAuthenticator, + time: time.Time{}, + want: ptr(time.Minute * 15), + }, + { + desc: "update last to be zero time records nil", + auth: secondAuthenticator, + time: time.Time{}, + want: nil, + }, + } { + // Must run in series as the tests build off each other. + t.Run(tc.desc, func(t *testing.T) { + tracker.set(tc.auth, tc.time) + tracker.report(nowFn) + if mockMetric.v != nil && tc.want != nil { + if mockMetric.v.Seconds() != tc.want.Seconds() { + t.Errorf("got: %v; want: %v", mockMetric.v, tc.want) + } + } else if mockMetric.v != tc.want { + t.Errorf("got: %v; want: %v", mockMetric.v, tc.want) + } + }) + } +} diff --git a/staging/src/k8s.io/client-go/tools/metrics/metrics.go b/staging/src/k8s.io/client-go/tools/metrics/metrics.go index a01306c65dfa..ee6800f31ef3 100644 --- a/staging/src/k8s.io/client-go/tools/metrics/metrics.go +++ b/staging/src/k8s.io/client-go/tools/metrics/metrics.go @@ -26,6 +26,16 @@ import ( var registerMetrics sync.Once +// DurationMetric is a measurement of some amount of time. +type DurationMetric interface { + Observe(duration time.Duration) +} + +// TTLMetric sets the time to live of something. +type TTLMetric interface { + Set(ttl *time.Duration) +} + // LatencyMetric observes client latency partitioned by verb and url. type LatencyMetric interface { Observe(verb string, u url.URL, latency time.Duration) @@ -37,21 +47,51 @@ type ResultMetric interface { } var ( + // ClientCertTTL is the time to live of a client certificate + ClientCertTTL TTLMetric = noopTTL{} + // ClientCertRotationAge is the age of a certificate that has just been rotated. + ClientCertRotationAge DurationMetric = noopDuration{} // RequestLatency is the latency metric that rest clients will update. RequestLatency LatencyMetric = noopLatency{} // RequestResult is the result metric that rest clients will update. RequestResult ResultMetric = noopResult{} ) +// RegisterOpts contains all the metrics to register. Metrics may be nil. +type RegisterOpts struct { + ClientCertTTL TTLMetric + ClientCertRotationAge DurationMetric + RequestLatency LatencyMetric + RequestResult ResultMetric +} + // Register registers metrics for the rest client to use. This can // only be called once. -func Register(lm LatencyMetric, rm ResultMetric) { +func Register(opts RegisterOpts) { registerMetrics.Do(func() { - RequestLatency = lm - RequestResult = rm + if opts.ClientCertTTL != nil { + ClientCertTTL = opts.ClientCertTTL + } + if opts.ClientCertRotationAge != nil { + ClientCertRotationAge = opts.ClientCertRotationAge + } + if opts.RequestLatency != nil { + RequestLatency = opts.RequestLatency + } + if opts.RequestResult != nil { + RequestResult = opts.RequestResult + } }) } +type noopDuration struct{} + +func (noopDuration) Observe(time.Duration) {} + +type noopTTL struct{} + +func (noopTTL) Set(*time.Duration) {} + type noopLatency struct{} func (noopLatency) Observe(string, url.URL, time.Duration) {} diff --git a/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go b/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go index ac0c9eb33195..51888a640ecc 100644 --- a/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go +++ b/staging/src/k8s.io/component-base/metrics/prometheus/restclient/metrics.go @@ -17,6 +17,7 @@ limitations under the License. package restclient import ( + "math" "net/url" "time" @@ -55,13 +56,62 @@ var ( }, []string{"code", "method", "host"}, ) + + execPluginCertTTL = k8smetrics.NewGauge( + &k8smetrics.GaugeOpts{ + Name: "rest_client_exec_plugin_ttl_seconds", + Help: "Gauge of the shortest TTL (time-to-live) of the client " + + "certificate(s) managed by the auth exec plugin. The value " + + "is in seconds until certificate expiry. If auth exec " + + "plugins are unused or manage no TLS certificates, the " + + "value will be +INF.", + }, + ) + + execPluginCertRotation = k8smetrics.NewHistogram( + &k8smetrics.HistogramOpts{ + Name: "rest_client_exec_plugin_certificate_rotation_age", + Help: "Histogram of the number of seconds the last auth exec " + + "plugin client certificate lived before being rotated. " + + "If auth exec plugin client certificates are unused, " + + "histogram will contain no data.", + // There are three sets of ranges these buckets intend to capture: + // - 10-60 minutes: captures a rotation cadence which is + // happening too quickly. + // - 4 hours - 1 month: captures an ideal rotation cadence. + // - 3 months - 4 years: captures a rotation cadence which is + // is probably too slow or much too slow. + Buckets: []float64{ + 600, // 10 minutes + 1800, // 30 minutes + 3600, // 1 hour + 14400, // 4 hours + 86400, // 1 day + 604800, // 1 week + 2592000, // 1 month + 7776000, // 3 months + 15552000, // 6 months + 31104000, // 1 year + 124416000, // 4 years + }, + }, + ) ) func init() { + execPluginCertTTL.Set(math.Inf(1)) // Initialize TTL to +INF + legacyregistry.MustRegister(requestLatency) legacyregistry.MustRegister(deprecatedRequestLatency) legacyregistry.MustRegister(requestResult) - metrics.Register(&latencyAdapter{m: requestLatency, dm: deprecatedRequestLatency}, &resultAdapter{requestResult}) + legacyregistry.MustRegister(execPluginCertTTL) + legacyregistry.MustRegister(execPluginCertRotation) + metrics.Register(metrics.RegisterOpts{ + ClientCertTTL: &ttlAdapter{m: execPluginCertTTL}, + ClientCertRotationAge: &rotationAdapter{m: execPluginCertRotation}, + RequestLatency: &latencyAdapter{m: requestLatency, dm: deprecatedRequestLatency}, + RequestResult: &resultAdapter{requestResult}, + }) } type latencyAdapter struct { @@ -81,3 +131,23 @@ type resultAdapter struct { func (r *resultAdapter) Increment(code, method, host string) { r.m.WithLabelValues(code, method, host).Inc() } + +type ttlAdapter struct { + m *k8smetrics.Gauge +} + +func (e *ttlAdapter) Set(ttl *time.Duration) { + if ttl == nil { + e.m.Set(math.Inf(1)) + } else { + e.m.Set(float64(ttl.Seconds())) + } +} + +type rotationAdapter struct { + m *k8smetrics.Histogram +} + +func (r *rotationAdapter) Observe(d time.Duration) { + r.m.Observe(d.Seconds()) +}