Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

kubeadm: graduate kubelet start join phase #73732

Merged
merged 1 commit into from
Feb 6, 2019
Merged

kubeadm: graduate kubelet start join phase #73732

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 0 additions & 4 deletions cmd/kubeadm/app/cmd/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,7 @@ go_library(
"//cmd/kubeadm/app/phases/certs:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
"//cmd/kubeadm/app/phases/kubelet:go_default_library",
"//cmd/kubeadm/app/phases/markcontrolplane:go_default_library",
"//cmd/kubeadm/app/phases/patchnode:go_default_library",
"//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util:go_default_library",
Expand All @@ -59,11 +57,9 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/duration:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/version:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//staging/src/k8s.io/cluster-bootstrap/token/api:go_default_library",
"//staging/src/k8s.io/cluster-bootstrap/token/util:go_default_library",
"//vendor/github.com/lithammer/dedent:go_default_library",
Expand Down
126 changes: 2 additions & 124 deletions cmd/kubeadm/app/cmd/join.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright 2016 The Kubernetes Authors.
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -28,10 +28,8 @@ import (
"github.com/spf13/cobra"
flag "github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
clientset "k8s.io/client-go/kubernetes"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
certutil "k8s.io/client-go/util/cert"
"k8s.io/klog"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
Expand All @@ -44,16 +42,11 @@ import (
kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
"k8s.io/kubernetes/cmd/kubeadm/app/discovery"
etcdphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/etcd"
kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
markcontrolplanephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markcontrolplane"
patchnodephase "k8s.io/kubernetes/cmd/kubeadm/app/phases/patchnode"
uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
utilsexec "k8s.io/utils/exec"
)

var (
Expand Down Expand Up @@ -127,19 +120,6 @@ var (
Often times the same token is used for both parts. In this case, the
--token flag can be used instead of specifying each token individually.
`)

kubeadmJoinFailMsg = dedent.Dedent(`
Unfortunately, an error has occurred:
%v

This error is likely caused by:
- The kubelet is not running
- The kubelet is unhealthy due to a misconfiguration of the node in some way (required cgroups disabled)

If you are on a systemd-powered system, you can try to troubleshoot the error with the following commands:
- 'systemctl status kubelet'
- 'journalctl -xeu kubelet'
`)
)

// joinOptions defines all the options exposed via flags by kubeadm join.
Expand Down Expand Up @@ -201,6 +181,7 @@ func NewCmdJoin(out io.Writer, joinOptions *joinOptions) *cobra.Command {
joinRunner.AppendPhase(phases.NewPreflightPhase())
joinRunner.AppendPhase(phases.NewControlPlanePreparePhase())
joinRunner.AppendPhase(phases.NewCheckEtcdPhase())
joinRunner.AppendPhase(phases.NewKubeletStartPhase())

// sets the data builder function, that will be used by the runner
// both when running the entire workflow or single phases
Expand Down Expand Up @@ -438,25 +419,11 @@ func (j *joinData) Run() error {
// TODO: individual phases should call these:
// - phases that need initCfg should call joinData.InitCfg().
// - phases that need tlsBootstrapCfg should call joinData.TLSBootstrapCfg().
tlsBootstrapCfg, err := j.TLSBootstrapCfg()
if err != nil {
return err
}
initCfg, err := j.InitCfg()
if err != nil {
return err
}

// Executes the kubelet TLS bootstrap process, that completes with the node
// joining the cluster with a dedicates set of credentials as required by
// the node authorizer.
// if the node is hosting a new control plane instance, since it uses static pods for the control plane,
// as soon as the kubelet starts it will take charge of creating control plane
// components on the node.
if err := j.BootstrapKubelet(tlsBootstrapCfg, initCfg); err != nil {
return err
}

// if the node is hosting a new control plane instance
if j.cfg.ControlPlane != nil {
// Completes the control plane setup
Expand Down Expand Up @@ -485,81 +452,6 @@ func (j *joinData) Run() error {
return nil
}

// BootstrapKubelet executes the kubelet TLS bootstrap process.
// This process is executed by the kubelet and completes with the node joining the cluster
// with a dedicates set of credentials as required by the node authorizer
func (j *joinData) BootstrapKubelet(tlsBootstrapCfg *clientcmdapi.Config, initConfiguration *kubeadmapi.InitConfiguration) error {
bootstrapKubeConfigFile := kubeadmconstants.GetBootstrapKubeletKubeConfigPath()

// Write the bootstrap kubelet config file or the TLS-Boostrapped kubelet config file down to disk
klog.V(1).Infoln("[join] writing bootstrap kubelet config file at", bootstrapKubeConfigFile)
if err := kubeconfigutil.WriteToDisk(bootstrapKubeConfigFile, tlsBootstrapCfg); err != nil {
return errors.Wrap(err, "couldn't save bootstrap-kubelet.conf to disk")
}

// Write the ca certificate to disk so kubelet can use it for authentication
cluster := tlsBootstrapCfg.Contexts[tlsBootstrapCfg.CurrentContext].Cluster
if _, err := os.Stat(j.cfg.CACertPath); os.IsNotExist(err) {
if err := certutil.WriteCert(j.cfg.CACertPath, tlsBootstrapCfg.Clusters[cluster].CertificateAuthorityData); err != nil {
return errors.Wrap(err, "couldn't save the CA certificate to disk")
}
}

kubeletVersion, err := preflight.GetKubeletVersion(utilsexec.New())
if err != nil {
return err
}

bootstrapClient, err := kubeconfigutil.ClientSetFromFile(bootstrapKubeConfigFile)
if err != nil {
return errors.Errorf("couldn't create client from kubeconfig file %q", bootstrapKubeConfigFile)
}

// Configure the kubelet. In this short timeframe, kubeadm is trying to stop/restart the kubelet
// Try to stop the kubelet service so no race conditions occur when configuring it
klog.V(1).Infof("Stopping the kubelet")
kubeletphase.TryStopKubelet()

// Write the configuration for the kubelet (using the bootstrap token credentials) to disk so the kubelet can start
if err := kubeletphase.DownloadConfig(bootstrapClient, kubeletVersion, kubeadmconstants.KubeletRunDirectory); err != nil {
return err
}

// Write env file with flags for the kubelet to use. We only want to
// register the joining node with the specified taints if the node
// is not a master. The markmaster phase will register the taints otherwise.
registerTaintsUsingFlags := j.cfg.ControlPlane == nil
if err := kubeletphase.WriteKubeletDynamicEnvFile(&initConfiguration.ClusterConfiguration, &initConfiguration.NodeRegistration, registerTaintsUsingFlags, kubeadmconstants.KubeletRunDirectory); err != nil {
return err
}

// Try to start the kubelet service in case it's inactive
klog.V(1).Infof("Starting the kubelet")
kubeletphase.TryStartKubelet()

// Now the kubelet will perform the TLS Bootstrap, transforming /etc/kubernetes/bootstrap-kubelet.conf to /etc/kubernetes/kubelet.conf
// Wait for the kubelet to create the /etc/kubernetes/kubelet.conf kubeconfig file. If this process
// times out, display a somewhat user-friendly message.
waiter := apiclient.NewKubeWaiter(nil, kubeadmconstants.TLSBootstrapTimeout, os.Stdout)
if err := waiter.WaitForKubeletAndFunc(waitForTLSBootstrappedClient); err != nil {
fmt.Printf(kubeadmJoinFailMsg, err)
return err
}

// When we know the /etc/kubernetes/kubelet.conf file is available, get the client
client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath())
if err != nil {
return err
}

klog.V(1).Infof("[join] preserving the crisocket information for the node")
if err := patchnodephase.AnnotateCRISocket(client, j.cfg.NodeRegistration.Name, j.cfg.NodeRegistration.CRISocket); err != nil {
return errors.Wrap(err, "error uploading crisocket")
}

return nil
}

// PostInstallControlPlane marks the new node as master and update the cluster status with information about current node
func (j *joinData) PostInstallControlPlane(initConfiguration *kubeadmapi.InitConfiguration) error {
kubeConfigFile := filepath.Join(kubeadmconstants.KubernetesDir, kubeadmconstants.AdminKubeConfigFileName)
Expand Down Expand Up @@ -599,20 +491,6 @@ func (j *joinData) PostInstallControlPlane(initConfiguration *kubeadmapi.InitCon
return nil
}

// waitForTLSBootstrappedClient waits for the /etc/kubernetes/kubelet.conf file to be available
func waitForTLSBootstrappedClient() error {
fmt.Println("[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...")

// Loop on every falsy return. Return with an error if raised. Exit successfully if true is returned.
return wait.PollImmediate(kubeadmconstants.APICallRetryInterval, kubeadmconstants.TLSBootstrapTimeout, func() (bool, error) {
// Check that we can create a client set out of the kubelet kubeconfig. This ensures not
// only that the kubeconfig file exists, but that other files required by it also exist (like
// client certificate and key)
_, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetKubeletKubeConfigPath())
return (err == nil), nil
})
}

// fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery
func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
// Retrieves the kubeadm configuration
Expand Down
4 changes: 2 additions & 2 deletions cmd/kubeadm/app/cmd/phases/init/kubelet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -29,7 +29,7 @@ import (
var (
kubeletStartPhaseExample = normalizer.Examples(`
# Writes a dynamic environment file with kubelet flags from a InitConfiguration file.
kubeadm init phase kubelet-start --config masterconfig.yaml
kubeadm init phase kubelet-start --config config.yaml
`)
)

Expand Down
8 changes: 8 additions & 0 deletions cmd/kubeadm/app/cmd/phases/join/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go_library(
srcs = [
"checketcd.go",
"controlplane.go",
"kubelet.go",
"preflight.go",
],
importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/join",
Expand All @@ -19,10 +20,17 @@ go_library(
"//cmd/kubeadm/app/phases/controlplane:go_default_library",
"//cmd/kubeadm/app/phases/etcd:go_default_library",
"//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
"//cmd/kubeadm/app/phases/kubelet:go_default_library",
"//cmd/kubeadm/app/phases/patchnode:go_default_library",
"//cmd/kubeadm/app/preflight:go_default_library",
"//cmd/kubeadm/app/util/apiclient:go_default_library",
"//cmd/kubeadm/app/util/kubeconfig:go_default_library",
"//pkg/util/normalizer:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/tools/clientcmd/api:go_default_library",
"//staging/src/k8s.io/client-go/util/cert:go_default_library",
"//vendor/github.com/lithammer/dedent:go_default_library",
"//vendor/github.com/pkg/errors:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
Expand Down
2 changes: 0 additions & 2 deletions cmd/kubeadm/app/cmd/phases/join/controlplane.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (

"github.com/pkg/errors"

clientset "k8s.io/client-go/kubernetes"
kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases/workflow"
Expand All @@ -34,7 +33,6 @@ import (

type controlPlanePrepareData interface {
Cfg() *kubeadmapi.JoinConfiguration
ClientSetFromFile(string) (*clientset.Clientset, error)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I removed this because it's not used, and IMO we no need a pr only for that.

InitCfg() (*kubeadmapi.InitConfiguration, error)
}

Expand Down