Skip to content

Commit

Permalink
Merge pull request #41912 from jcbsmpsn/rotate-client-certificate
Browse files Browse the repository at this point in the history 
Automatic merge from submit-queue (batch tested with PRs 46726, 41912, 46695, 46034, 46551)

Rotate kubelet client certificate.

Changes the kubelet so it bootstraps off the cert/key specified in the
config file and uses those to request new cert/key pairs from the
Certificate Signing Request API, as well as rotating client certificates
when they approach expiration.

Default behavior is for client certificate rotation to be disabled. If enabled
using a command line flag, the kubelet exits each time the certificate is
rotated. I tried to use `GetCertificate` in [tls.Config](https://golang.org/pkg/crypto/tls/#Config) but it is only called
on the server side of connections. Then I tried `GetClientCertificate`,
but it is new in 1.8.

**Release note**
```release-note
With --feature-gates=RotateKubeletClientCertificate=true set, the kubelet will
request a client certificate from the API server during the boot cycle and pause
waiting for the request to be satisfied. It will continually refresh the certificate
as the certificates expiration approaches.
```
  • Loading branch information
Kubernetes Submit Queue committed Jun 3, 2017
2 parents aab12f2 + 1519bb9 commit 24d0997
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 8 deletions.
3 changes: 3 additions & 0 deletions cmd/kubelet/app/BUILD
View file
Expand Up @@ -37,6 +37,7 @@ go_library(
"//cmd/kubelet/app/options:go_default_library",
"//pkg/api:go_default_library",
"//pkg/api/v1:go_default_library",
"//pkg/apis/certificates/v1beta1:go_default_library",
"//pkg/apis/componentconfig:go_default_library",
"//pkg/apis/componentconfig/v1alpha1:go_default_library",
"//pkg/capabilities:go_default_library",
Expand All @@ -52,6 +53,7 @@ go_library(
"//pkg/features:go_default_library",
"//pkg/kubelet:go_default_library",
"//pkg/kubelet/cadvisor:go_default_library",
"//pkg/kubelet/certificate:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/config:go_default_library",
"//pkg/kubelet/container:go_default_library",
Expand Down Expand Up @@ -110,6 +112,7 @@ go_library(
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
Expand Down
109 changes: 109 additions & 0 deletions cmd/kubelet/app/server.go
View file
Expand Up @@ -19,6 +19,8 @@ package app

import (
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"errors"
"fmt"
"io/ioutil"
Expand All @@ -41,6 +43,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
Expand All @@ -58,6 +61,7 @@ import (
"k8s.io/kubernetes/cmd/kubelet/app/options"
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/api/v1"
certificates "k8s.io/kubernetes/pkg/apis/certificates/v1beta1"
"k8s.io/kubernetes/pkg/apis/componentconfig"
componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
"k8s.io/kubernetes/pkg/capabilities"
Expand All @@ -68,6 +72,7 @@ import (
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/kubelet"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/certificate"
"k8s.io/kubernetes/pkg/kubelet/cm"
"k8s.io/kubernetes/pkg/kubelet/config"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
Expand Down Expand Up @@ -446,10 +451,30 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
}

clientConfig, err := CreateAPIServerClientConfig(s)

var clientCertificateManager certificate.Manager
if err == nil {
if utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
nodeName, err := getNodeName(cloud, nodeutil.GetHostname(s.HostnameOverride))
if err != nil {
return err
}
clientCertificateManager, err = initializeClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData)
if err != nil {
return err
}
if err := updateTransport(clientConfig, clientCertificateManager); err != nil {
return err
}
}

kubeClient, err = clientset.NewForConfig(clientConfig)
if err != nil {
glog.Warningf("New kubeClient from clientConfig error: %v", err)
} else if kubeClient.Certificates() != nil && clientCertificateManager != nil {
glog.V(2).Info("Starting client certificate rotation.")
clientCertificateManager.SetCertificateSigningRequestClient(kubeClient.Certificates().CertificateSigningRequests())
clientCertificateManager.Start()
}
externalKubeClient, err = clientgoclientset.NewForConfig(clientConfig)
if err != nil {
Expand Down Expand Up @@ -599,6 +624,90 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
return nil
}

func updateTransport(clientConfig *restclient.Config, clientCertificateManager certificate.Manager) error {
if clientConfig.Transport != nil {
return fmt.Errorf("there is already a transport configured")
}
tlsConfig, err := restclient.TLSConfigFor(clientConfig)
if err != nil {
return fmt.Errorf("unable to configure TLS for the rest client: %v", err)
}
if tlsConfig == nil {
tlsConfig = &tls.Config{}
}
tlsConfig.Certificates = nil
tlsConfig.GetClientCertificate = func(requestInfo *tls.CertificateRequestInfo) (*tls.Certificate, error) {
cert := clientCertificateManager.Current()
if cert == nil {
return &tls.Certificate{Certificate: nil}, nil
}
return cert, nil
}
clientConfig.Transport = utilnet.SetTransportDefaults(&http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: tlsConfig,
MaxIdleConnsPerHost: 25,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
})
clientConfig.CertData = nil
clientConfig.KeyData = nil
clientConfig.CertFile = ""
clientConfig.KeyFile = ""
clientConfig.CAData = nil
clientConfig.CAFile = ""
return nil
}

// initializeClientCertificateManager sets up a certificate manager without a
// client that can be used to sign new certificates (or rotate). It answers with
// whatever certificate it is initialized with. If a CSR client is set later, it
// may begin rotating/renewing the client cert
func initializeClientCertificateManager(certDirectory string, nodeName types.NodeName, certData []byte, keyData []byte) (certificate.Manager, error) {
certificateStore, err := certificate.NewFileStore(
"kubelet-client",
certDirectory,
certDirectory,
"",
"")
if err != nil {
return nil, fmt.Errorf("failed to initialize certificate store: %v", err)
}
clientCertificateManager, err := certificate.NewManager(&certificate.Config{
Template: &x509.CertificateRequest{
Subject: pkix.Name{
Organization: []string{"system:nodes"},
CommonName: fmt.Sprintf("system:node:%s", nodeName),
},
},
Usages: []certificates.KeyUsage{
// https://tools.ietf.org/html/rfc5280#section-4.2.1.3
//
// DigitalSignature allows the certificate to be used to verify
// digital signatures including signatures used during TLS
// negotiation.
certificates.UsageDigitalSignature,
// KeyEncipherment allows the cert/key pair to be used to encrypt
// keys, including the symetric keys negotiated during TLS setup
// and used for data transfer..
certificates.UsageKeyEncipherment,
// ClientAuth allows the cert to be used by a TLS client to
// authenticate itself to the TLS server.
certificates.UsageClientAuth,
},
CertificateStore: certificateStore,
BootstrapCertificatePEM: certData,
BootstrapKeyPEM: keyData,
})
if err != nil {
return nil, fmt.Errorf("failed to initialize certificate manager: %v", err)
}
return clientCertificateManager, nil
}

// getNodeName returns the node name according to the cloud provider
// if cloud provider is specified. Otherwise, returns the hostname of the node.
func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) {
Expand Down
8 changes: 8 additions & 0 deletions pkg/features/kube_features.go
View file
Expand Up @@ -97,6 +97,13 @@ const (
// certificate as expiration approaches.
RotateKubeletServerCertificate utilfeature.Feature = "RotateKubeletServerCertificate"

// owner: @jcbsmpsn
// alpha: v1.7
//
// Automatically renews the client certificate used for communicating with
// the API server as the certificate approaches expiration.
RotateKubeletClientCertificate utilfeature.Feature = "RotateKubeletClientCertificate"

// owner: @msau
// alpha: v1.7
//
Expand Down Expand Up @@ -128,6 +135,7 @@ var defaultKubernetesFeatureGates = map[utilfeature.Feature]utilfeature.FeatureS
Accelerators: {Default: false, PreRelease: utilfeature.Alpha},
TaintBasedEvictions: {Default: false, PreRelease: utilfeature.Alpha},
RotateKubeletServerCertificate: {Default: false, PreRelease: utilfeature.Alpha},
RotateKubeletClientCertificate: {Default: false, PreRelease: utilfeature.Alpha},
PersistentLocalVolumes: {Default: false, PreRelease: utilfeature.Alpha},
LocalStorageCapacityIsolation: {Default: false, PreRelease: utilfeature.Alpha},

Expand Down
10 changes: 7 additions & 3 deletions pkg/kubelet/certificate/certificate_manager.go
View file
Expand Up @@ -52,14 +52,17 @@ type Manager interface {
// Start the API server status sync loop.
Start()
// Current returns the currently selected certificate from the
// certificate manager.
// certificate manager, as well as the associated certificate and key data
// in PEM format.
Current() *tls.Certificate
}

// Config is the set of configuration parameters available for a new Manager.
type Config struct {
// CertificateSigningRequestClient will be used for signing new certificate
// requests generated when a key rotation occurs.
// requests generated when a key rotation occurs. It must be set either at
// initialization or by using CertificateSigningRequestClient before
// Manager.Start() is called.
CertificateSigningRequestClient certificatesclient.CertificateSigningRequestInterface
// Template is the CertificateRequest that will be used as a template for
// generating certificate signing requests for all new keys generated as
Expand Down Expand Up @@ -99,7 +102,8 @@ type Config struct {
// Depending on the concrete implementation, the backing store for this
// behavior may vary.
type Store interface {
// Current returns the currently selected certificate. If the Store doesn't
// Current returns the currently selected certificate, as well as the
// associated certificate and key data in PEM format. If the Store doesn't
// have a cert/key pair currently, it should return a NoCertKeyError so
// that the Manager can recover by using bootstrap certificates to request
// a new cert/key pair.
Expand Down
8 changes: 3 additions & 5 deletions pkg/kubelet/certificate/certificate_manager_test.go
View file
Expand Up @@ -173,7 +173,7 @@ func TestShouldRotate(t *testing.T) {
}
m.setRotationDeadline()
if m.shouldRotate() != test.shouldRotate {
t.Errorf("For time %v, a certificate issued for (%v, %v) should rotate should be %t.",
t.Errorf("Time %v, a certificate issued for (%v, %v) should rotate should be %t.",
now,
m.cert.Leaf.NotBefore,
m.cert.Leaf.NotAfter,
Expand Down Expand Up @@ -296,7 +296,6 @@ func TestNewManagerBootstrap(t *testing.T) {
if cert == nil {
t.Errorf("Certificate was nil, expected something.")
}

if m, ok := cm.(*manager); !ok {
t.Errorf("Expected a '*manager' from 'NewManager'")
} else if !m.shouldRotate() {
Expand Down Expand Up @@ -335,7 +334,6 @@ func TestNewManagerNoBootstrap(t *testing.T) {
if currentCert == nil {
t.Errorf("Certificate was nil, expected something.")
}

if m, ok := cm.(*manager); !ok {
t.Errorf("Expected a '*manager' from 'NewManager'")
} else {
Expand Down Expand Up @@ -386,8 +384,8 @@ func TestGetCurrentCertificateOrBootstrap(t *testing.T) {
store,
tc.bootstrapCertData,
tc.bootstrapKeyData)
if certResult == nil || tc.expectedCert == nil {
if certResult != tc.expectedCert {
if certResult == nil || certResult.Certificate == nil || tc.expectedCert == nil {
if certResult != nil && tc.expectedCert != nil {
t.Errorf("Got certificate %v, wanted %v", certResult, tc.expectedCert)
}
} else {
Expand Down

0 comments on commit 24d0997

Please sign in to comment.