Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Revert "Make bootstrap client cert loading part of rotation" #71173

Merged
merged 1 commit into from
Nov 17, 2018
Merged

Revert "Make bootstrap client cert loading part of rotation" #71173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
17 changes: 1 addition & 16 deletions cmd/kubelet/app/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,22 +8,8 @@ load(

go_test(
name = "go_default_test",
srcs = [
"server_bootstrap_test.go",
"server_test.go",
],
srcs = ["server_test.go"],
embed = [":go_default_library"],
deps = [
"//staging/src/k8s.io/api/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//vendor/github.com/cloudflare/cfssl/config:go_default_library",
"//vendor/github.com/cloudflare/cfssl/signer:go_default_library",
"//vendor/github.com/cloudflare/cfssl/signer/local:go_default_library",
],
)

go_library(
Expand Down Expand Up @@ -127,7 +113,6 @@ go_library(
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/authentication/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/certificates/v1beta1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
"//staging/src/k8s.io/client-go/rest:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd:go_default_library",
Expand Down
156 changes: 59 additions & 97 deletions cmd/kubelet/app/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ import (
"k8s.io/apiserver/pkg/util/flag"
"k8s.io/client-go/dynamic"
clientset "k8s.io/client-go/kubernetes"
certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
Expand Down Expand Up @@ -538,39 +537,66 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
return err
}

if s.BootstrapKubeconfig != "" {
if err := bootstrap.LoadClientCert(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil {
return err
}
}

// if in standalone mode, indicate as much by setting all clients to nil
switch {
case standaloneMode:
if standaloneMode {
kubeDeps.KubeClient = nil
kubeDeps.DynamicKubeClient = nil
kubeDeps.EventClient = nil
kubeDeps.HeartbeatClient = nil
klog.Warningf("standalone mode, no API client")
} else if kubeDeps.KubeClient == nil || kubeDeps.EventClient == nil || kubeDeps.HeartbeatClient == nil || kubeDeps.DynamicKubeClient == nil {
// initialize clients if not standalone mode and any of the clients are not provided
var kubeClient clientset.Interface
var eventClient v1core.EventsGetter
var heartbeatClient clientset.Interface
var dynamicKubeClient dynamic.Interface

clientConfig, err := createAPIServerClientConfig(s)
if err != nil {
return fmt.Errorf("invalid kubeconfig: %v", err)
}

case kubeDeps.KubeClient == nil, kubeDeps.EventClient == nil, kubeDeps.HeartbeatClient == nil, kubeDeps.DynamicKubeClient == nil:
clientConfig, closeAllConns, err := buildKubeletClientConfig(s, nodeName)
var clientCertificateManager certificate.Manager
if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
clientCertificateManager, err = kubeletcertificate.NewKubeletClientCertificateManager(s.CertDirectory, nodeName, clientConfig.CertData, clientConfig.KeyData, clientConfig.CertFile, clientConfig.KeyFile)
if err != nil {
return err
}
}
// we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable
// to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper
// or the bootstrapping credentials to potentially lay down new initial config.
closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, clientConfig, clientCertificateManager, 5*time.Minute)
if err != nil {
return err
}
kubeDeps.OnHeartbeatFailure = closeAllConns

kubeDeps.KubeClient, err = clientset.NewForConfig(clientConfig)
kubeClient, err = clientset.NewForConfig(clientConfig)
if err != nil {
return fmt.Errorf("failed to initialize kubelet client: %v", err)
klog.Warningf("New kubeClient from clientConfig error: %v", err)
} else if kubeClient.CertificatesV1beta1() != nil && clientCertificateManager != nil {
klog.V(2).Info("Starting client certificate rotation.")
clientCertificateManager.SetCertificateSigningRequestClient(kubeClient.CertificatesV1beta1().CertificateSigningRequests())
clientCertificateManager.Start()
}

kubeDeps.DynamicKubeClient, err = dynamic.NewForConfig(clientConfig)
dynamicKubeClient, err = dynamic.NewForConfig(clientConfig)
if err != nil {
return fmt.Errorf("failed to initialize kubelet dynamic client: %v", err)
klog.Warningf("Failed to initialize dynamic KubeClient: %v", err)
}

// make a separate client for events
eventClientConfig := *clientConfig
eventClientConfig.QPS = float32(s.EventRecordQPS)
eventClientConfig.Burst = int(s.EventBurst)
kubeDeps.EventClient, err = v1core.NewForConfig(&eventClientConfig)
eventClient, err = v1core.NewForConfig(&eventClientConfig)
if err != nil {
return fmt.Errorf("failed to initialize kubelet event client: %v", err)
klog.Warningf("Failed to create API Server client for Events: %v", err)
}

// make a separate client for heartbeat with throttling disabled and a timeout attached
Expand All @@ -584,18 +610,28 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
}
}
heartbeatClientConfig.QPS = float32(-1)
kubeDeps.HeartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig)
heartbeatClient, err = clientset.NewForConfig(&heartbeatClientConfig)
if err != nil {
return fmt.Errorf("failed to initialize kubelet heartbeat client: %v", err)
klog.Warningf("Failed to create API Server client for heartbeat: %v", err)
}

// CRDs are JSON only, and client renegotiation for streaming is not correct as per #67803
csiClientConfig := restclient.CopyConfig(clientConfig)
csiClientConfig.ContentType = "application/json"
kubeDeps.CSIClient, err = csiclientset.NewForConfig(csiClientConfig)
// csiClient works with CRDs that support json only
clientConfig.ContentType = "application/json"
csiClient, err := csiclientset.NewForConfig(clientConfig)
if err != nil {
return fmt.Errorf("failed to initialize kubelet storage client: %v", err)
klog.Warningf("Failed to create CSI API client: %v", err)
}

kubeDeps.KubeClient = kubeClient
kubeDeps.DynamicKubeClient = dynamicKubeClient
if heartbeatClient != nil {
kubeDeps.HeartbeatClient = heartbeatClient
kubeDeps.OnHeartbeatFailure = closeAllConns
}
if eventClient != nil {
kubeDeps.EventClient = eventClient
}
kubeDeps.CSIClient = csiClient
}

// If the kubelet config controller is available, and dynamic config is enabled, start the config and status sync loops
Expand Down Expand Up @@ -735,81 +771,6 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.Dependencies, stopCh <-chan
return nil
}

// buildKubeletClientConfig constructs the appropriate client config for the kubelet depending on whether
// bootstrapping is enabled or client certificate rotation is enabled.
func buildKubeletClientConfig(s *options.KubeletServer, nodeName types.NodeName) (*restclient.Config, func(), error) {
if s.RotateCertificates && utilfeature.DefaultFeatureGate.Enabled(features.RotateKubeletClientCertificate) {
certConfig, clientConfig, err := bootstrap.LoadClientConfig(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory)
if err != nil {
return nil, nil, err
}

clientCertificateManager, err := buildClientCertificateManager(certConfig, clientConfig, s.CertDirectory, nodeName)
if err != nil {
return nil, nil, err
}

// the rotating transport will use the cert from the cert manager instead of these files
transportConfig := restclient.CopyConfig(clientConfig)
transportConfig.CertFile = ""
transportConfig.KeyFile = ""

// we set exitAfter to five minutes because we use this client configuration to request new certs - if we are unable
// to request new certs, we will be unable to continue normal operation. Exiting the process allows a wrapper
// or the bootstrapping credentials to potentially lay down new initial config.
closeAllConns, err := kubeletcertificate.UpdateTransport(wait.NeverStop, transportConfig, clientCertificateManager, 5*time.Minute)
if err != nil {
return nil, nil, err
}

klog.V(2).Info("Starting client certificate rotation.")
clientCertificateManager.Start()

return transportConfig, closeAllConns, nil
}

if len(s.BootstrapKubeconfig) > 0 {
if err := bootstrap.LoadClientCert(s.KubeConfig, s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil {
return nil, nil, err
}
}

clientConfig, err := createAPIServerClientConfig(s)
if err != nil {
return nil, nil, fmt.Errorf("invalid kubeconfig: %v", err)
}
return clientConfig, nil, nil
}

// buildClientCertificateManager creates a certificate manager that will use certConfig to request a client certificate
// if no certificate is available, or the most recent clientConfig (which is assumed to point to the cert that the manager will
// write out).
func buildClientCertificateManager(certConfig, clientConfig *restclient.Config, certDir string, nodeName types.NodeName) (certificate.Manager, error) {
newClientFn := func(current *tls.Certificate) (certificatesclient.CertificateSigningRequestInterface, error) {
// If we have a valid certificate, use that to fetch CSRs. Otherwise use the bootstrap
// credentials. In the future it would be desirable to change the behavior of bootstrap
// to always fall back to the external bootstrap credentials when such credentials are
// provided by a fundamental trust system like cloud VM identity or an HSM module.
config := certConfig
if current != nil {
config = clientConfig
}
client, err := clientset.NewForConfig(config)
if err != nil {
return nil, err
}
return client.CertificatesV1beta1().CertificateSigningRequests(), nil
}

return kubeletcertificate.NewKubeletClientCertificateManager(
certDir,
nodeName,
clientConfig.CertFile,
clientConfig.KeyFile,
newClientFn,
)
}

// getNodeName returns the node name according to the cloud provider
// if cloud provider is specified. Otherwise, returns the hostname of the node.
func getNodeName(cloud cloudprovider.Interface, hostname string) (types.NodeName, error) {
Expand Down Expand Up @@ -908,10 +869,11 @@ func kubeconfigClientConfig(s *options.KubeletServer) (*restclient.Config, error
// createClientConfig creates a client configuration from the command line arguments.
// If --kubeconfig is explicitly set, it will be used.
func createClientConfig(s *options.KubeletServer) (*restclient.Config, error) {
if len(s.BootstrapKubeconfig) > 0 || len(s.KubeConfig) > 0 {
if s.BootstrapKubeconfig != "" || len(s.KubeConfig) > 0 {
return kubeconfigClientConfig(s)
} else {
return nil, fmt.Errorf("createClientConfig called in standalone mode")
}
return nil, fmt.Errorf("createClientConfig called in standalone mode")
}

// createAPIServerClientConfig generates a client.Config from command line flags
Expand Down