kubeadm: kustomize-ux

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -437,6 +437,7 @@ func isAllowedFlag(flagName string) bool {
 		kubeadmcmdoptions.NodeCRISocket,
 		kubeadmcmdoptions.KubeconfigDir,
 		kubeadmcmdoptions.UploadCerts,
+		kubeadmcmdoptions.Kustomize,
 		"print-join-command", "rootfs", "v")
 	if knownFlags.Has(flagName) {
 		return true

cmd/kubeadm/app/cmd/init.go

@@ -99,6 +99,7 @@ type initOptions struct {
 	externalClusterCfg      *kubeadmapiv1beta2.ClusterConfiguration
 	uploadCerts             bool
 	skipCertificateKeyPrint bool
+	kustomizeDir            string
 }
 
 // compile-time assert that the local data object satisfies the phases data interface.
@@ -120,6 +121,7 @@ type initData struct {
 	outputWriter            io.Writer
 	uploadCerts             bool
 	skipCertificateKeyPrint bool
+	kustomizeDir            string
 }
 
 // NewCmdInit returns "kubeadm init" command.
@@ -272,6 +274,7 @@ func AddInitOtherFlags(flagSet *flag.FlagSet, initOptions *initOptions) {
 		&initOptions.skipCertificateKeyPrint, options.SkipCertificateKeyPrint, initOptions.skipCertificateKeyPrint,
 		"Don't print the key used to encrypt the control-plane certificates.",
 	)
+	options.AddKustomizePodsFlag(flagSet, &initOptions.kustomizeDir)
 }
 
 // newInitOptions returns a struct ready for being used for creating cmd init flags.
@@ -404,6 +407,7 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		outputWriter:            out,
 		uploadCerts:             options.uploadCerts,
 		skipCertificateKeyPrint: options.skipCertificateKeyPrint,
+		kustomizeDir:            options.kustomizeDir,
 	}, nil
 }
 
@@ -532,6 +536,11 @@ func (d *initData) Tokens() []string {
 	return tokens
 }
 
+// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored
+func (d *initData) KustomizeDir() string {
+	return d.kustomizeDir
+}
+
 func printJoinCommand(out io.Writer, adminKubeConfigPath, token string, i *initData) error {
 	joinControlPlaneCommand, err := cmdutil.GetJoinControlPlaneCommand(adminKubeConfigPath, token, i.CertificateKey(), i.skipTokenPrint, i.skipCertificateKeyPrint)
 	if err != nil {

cmd/kubeadm/app/cmd/join.go

@@ -128,6 +128,7 @@ type joinOptions struct {
 	controlPlane          bool
 	ignorePreflightErrors []string
 	externalcfg           *kubeadmapiv1beta2.JoinConfiguration
+	kustomizeDir          string
 }
 
 // compile-time assert that the local data object satisfies the phases data interface.
@@ -142,6 +143,7 @@ type joinData struct {
 	clientSet             *clientset.Clientset
 	ignorePreflightErrors sets.String
 	outputWriter          io.Writer
+	kustomizeDir          string
 }
 
 // NewCmdJoin returns "kubeadm join" command.
@@ -276,6 +278,7 @@ func addJoinOtherFlags(flagSet *flag.FlagSet, joinOptions *joinOptions) {
 		&joinOptions.controlPlane, options.ControlPlane, joinOptions.controlPlane,
 		"Create a new control plane instance on this node",
 	)
+	options.AddKustomizePodsFlag(flagSet, &joinOptions.kustomizeDir)
 }
 
 // newJoinOptions returns a struct ready for being used for creating cmd join flags.
@@ -405,6 +408,7 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri
 		tlsBootstrapCfg:       tlsBootstrapCfg,
 		ignorePreflightErrors: ignorePreflightErrorsSet,
 		outputWriter:          out,
+		kustomizeDir:          opt.kustomizeDir,
 	}, nil
 }
 
@@ -470,6 +474,11 @@ func (j *joinData) OutputWriter() io.Writer {
 	return j.outputWriter
 }
 
+// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored
+func (j *joinData) KustomizeDir() string {
+	return j.kustomizeDir
+}
+
 // fetchInitConfigurationFromJoinConfiguration retrieves the init configuration from a join configuration, performing the discovery
 func fetchInitConfigurationFromJoinConfiguration(cfg *kubeadmapi.JoinConfiguration, tlsBootstrapCfg *clientcmdapi.Config) (*kubeadmapi.InitConfiguration, error) {
 	// Retrieves the kubeadm configuration

cmd/kubeadm/app/cmd/options/constant.go

@@ -142,4 +142,7 @@ const (
 
 	// EtcdUpgrade flag instruct kubeadm to execute etcd upgrade during upgrades
 	EtcdUpgrade = "etcd-upgrade"
+
+	// Kustomize flag sets the folder where kustomize patches for static pod manifest are stored
+	Kustomize = "experimental-kustomize"
 )

cmd/kubeadm/app/cmd/options/generic.go

@@ -88,3 +88,8 @@ func AddKubeadmOtherFlags(flagSet *pflag.FlagSet, rootfsPath *string) {
 		"[EXPERIMENTAL] The path to the 'real' host root filesystem.",
 	)
 }
+
+// AddKustomizePodsFlag adds the --kustomize flag to the given flagset
+func AddKustomizePodsFlag(fs *pflag.FlagSet, kustomizeDir *string) {
+	fs.StringVarP(kustomizeDir, Kustomize, "k", *kustomizeDir, "The path where kustomize patches for static pod manifests are stored.")
+}

cmd/kubeadm/app/cmd/phases/init/controlplane.go

@@ -100,6 +100,7 @@ func getControlPlanePhaseFlags(name string) []string {
 		options.CertificatesDir,
 		options.KubernetesVersion,
 		options.ImageRepository,
+		options.Kustomize,
 	}
 	if name == "all" || name == kubeadmconstants.KubeAPIServer {
 		flags = append(flags,
@@ -144,10 +145,6 @@ func runControlPlaneSubphase(component string) func(c workflow.RunData) error {
 		cfg := data.Cfg()
 
 		fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
-
-		// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
-		kustomizeDir := ""
-
-		return controlplane.CreateStaticPodFiles(data.ManifestDir(), kustomizeDir, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
+		return controlplane.CreateStaticPodFiles(data.ManifestDir(), data.KustomizeDir(), &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint, component)
 	}
 }

cmd/kubeadm/app/cmd/phases/init/data.go

@@ -45,4 +45,5 @@ type InitData interface {
 	OutputWriter() io.Writer
 	Client() (clientset.Interface, error)
 	Tokens() []string
+	KustomizeDir() string
 }

cmd/kubeadm/app/cmd/phases/init/data_test.go

@@ -48,3 +48,4 @@ func (t *testInitData) ExternalCA() bool                     { return false }
 func (t *testInitData) OutputWriter() io.Writer              { return nil }
 func (t *testInitData) Client() (clientset.Interface, error) { return nil, nil }
 func (t *testInitData) Tokens() []string                     { return nil }
+func (t *testInitData) KustomizeDir() string                 { return "" }

cmd/kubeadm/app/cmd/phases/init/etcd.go

@@ -69,6 +69,7 @@ func getEtcdPhaseFlags() []string {
 		options.CertificatesDir,
 		options.CfgPath,
 		options.ImageRepository,
+		options.Kustomize,
 	}
 	return flags
 }
@@ -92,11 +93,7 @@ func runEtcdPhaseLocal() func(c workflow.RunData) error {
 				fmt.Printf("[dryrun] Would ensure that %q directory is present\n", cfg.Etcd.Local.DataDir)
 			}
 			fmt.Printf("[etcd] Creating static Pod manifest for local etcd in %q\n", data.ManifestDir())
-
-			// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
-			kustomizeDir := ""
-
-			if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), kustomizeDir, cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
+			if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(data.ManifestDir(), data.KustomizeDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
 				return errors.Wrap(err, "error creating local etcd static pod manifest file")
 			}
 		} else {

cmd/kubeadm/app/cmd/phases/join/controlplanejoin.go

@@ -42,6 +42,9 @@ func getControlPlaneJoinPhaseFlags(name string) []string {
 		options.ControlPlane,
 		options.NodeName,
 	}
+	if name == "etcd" {
+		flags = append(flags, options.Kustomize)
+	}
 	if name != "mark-control-plane" {
 		flags = append(flags, options.APIServerAdvertiseAddress)
 	}
@@ -137,7 +140,7 @@ func runEtcdPhase(c workflow.RunData) error {
 	// "If you add a new member to a 1-node cluster, the cluster cannot make progress before the new member starts
 	// because it needs two members as majority to agree on the consensus. You will only see this behavior between the time
 	// etcdctl member add informs the cluster about the new member and the new member successfully establishing a connection to the 	// existing one."
-	if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
+	if err := etcdphase.CreateStackedEtcdStaticPodManifestFile(client, kubeadmconstants.GetStaticPodDirectory(), data.KustomizeDir(), cfg.NodeRegistration.Name, &cfg.ClusterConfiguration, &cfg.LocalAPIEndpoint); err != nil {
 		return errors.Wrap(err, "error creating local etcd static pod manifest file")
 	}
 

cmd/kubeadm/app/cmd/phases/join/controlplaneprepare.go

@@ -78,6 +78,7 @@ func getControlPlanePreparePhaseFlags(name string) []string {
 			options.TLSBootstrapToken,
 			options.TokenStr,
 			options.CertificateKey,
+			options.Kustomize,
 		}
 	case "download-certs":
 		flags = []string{
@@ -122,6 +123,7 @@ func getControlPlanePreparePhaseFlags(name string) []string {
 			options.APIServerBindPort,
 			options.CfgPath,
 			options.ControlPlane,
+			options.Kustomize,
 		}
 	default:
 		flags = []string{}
@@ -183,15 +185,11 @@ func runControlPlanePrepareControlPlaneSubphase(c workflow.RunData) error {
 	}
 
 	fmt.Printf("[control-plane] Using manifest folder %q\n", kubeadmconstants.GetStaticPodDirectory())
-
-	// TODO: this should be replaced by a value from a flag in subsequent PR. see the POC https://github.com/kubernetes/kubernetes/pull/80580
-	kustomizeDir := ""
-
 	for _, component := range kubeadmconstants.ControlPlaneComponents {
 		fmt.Printf("[control-plane] Creating static Pod manifest for %q\n", component)
 		err := controlplane.CreateStaticPodFiles(
 			kubeadmconstants.GetStaticPodDirectory(),
-			kustomizeDir,
+			data.KustomizeDir(),
 			&cfg.ClusterConfiguration,
 			&cfg.LocalAPIEndpoint,
 			component,

cmd/kubeadm/app/cmd/phases/join/data.go

@@ -35,4 +35,5 @@ type JoinData interface {
 	ClientSet() (*clientset.Clientset, error)
 	IgnorePreflightErrors() sets.String
 	OutputWriter() io.Writer
+	KustomizeDir() string
 }

cmd/kubeadm/app/cmd/phases/join/data_test.go

@@ -38,3 +38,4 @@ func (j *testJoinData) InitCfg() (*kubeadmapi.InitConfiguration, error) { return
 func (j *testJoinData) ClientSet() (*clientset.Clientset, error)        { return nil, nil }
 func (j *testJoinData) IgnorePreflightErrors() sets.String              { return nil }
 func (j *testJoinData) OutputWriter() io.Writer                         { return nil }
+func (j *testJoinData) KustomizeDir() string                            { return "" }

cmd/kubeadm/app/cmd/phases/upgrade/node/controlplane.go

@@ -39,6 +39,7 @@ func NewControlPlane() workflow.Phase {
 			options.KubeconfigPath,
 			options.CertificateRenewal,
 			options.EtcdUpgrade,
+			options.Kustomize,
 		},
 	}
 	return phase
@@ -63,16 +64,17 @@ func runControlPlane() func(c workflow.RunData) error {
 		dryRun := data.DryRun()
 		etcdUpgrade := data.EtcdUpgrade()
 		renewCerts := data.RenewCerts()
+		kustomizeDir := data.KustomizeDir()
 
 		// Upgrade the control plane and etcd if installed on this node
 		fmt.Printf("[upgrade] Upgrading your Static Pod-hosted control plane instance to version %q...\n", cfg.KubernetesVersion)
 		if dryRun {
-			return upgrade.DryRunStaticPodUpgrade(cfg)
+			return upgrade.DryRunStaticPodUpgrade(kustomizeDir, cfg)
 		}
 
 		waiter := apiclient.NewKubeWaiter(data.Client(), upgrade.UpgradeManifestTimeout, os.Stdout)
 
-		if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts); err != nil {
+		if err := upgrade.PerformStaticPodUpgrade(client, waiter, cfg, etcdUpgrade, renewCerts, kustomizeDir); err != nil {
 			return errors.Wrap(err, "couldn't complete the static pod upgrade")
 		}
 

cmd/kubeadm/app/cmd/phases/upgrade/node/data.go

@@ -31,4 +31,5 @@ type Data interface {
 	Cfg() *kubeadmapi.InitConfiguration
 	IsControlPlaneNode() bool
 	Client() clientset.Interface
+	KustomizeDir() string
 }

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -51,6 +51,7 @@ type applyFlags struct {
 	renewCerts         bool
 	criSocket          string
 	imagePullTimeout   time.Duration
+	kustomizeDir       string
 }
 
 // sessionIsInteractive returns true if the session is of an interactive type (the default, can be opted out of with -y, -f or --dry-run)
@@ -90,6 +91,7 @@ func NewCmdApply(apf *applyPlanFlags) *cobra.Command {
 	cmd.Flags().BoolVar(&flags.etcdUpgrade, "etcd-upgrade", flags.etcdUpgrade, "Perform the upgrade of etcd.")
 	cmd.Flags().BoolVar(&flags.renewCerts, "certificate-renewal", flags.renewCerts, "Perform the renewal of certificates used by component changed during upgrades.")
 	cmd.Flags().DurationVar(&flags.imagePullTimeout, "image-pull-timeout", flags.imagePullTimeout, "The maximum amount of time to wait for the control plane pods to be downloaded.")
+	options.AddKustomizePodsFlag(cmd.Flags(), &flags.kustomizeDir)
 
 	// The CRI socket flag is deprecated here, since it should be taken from the NodeRegistrationOptions for the current
 	// node instead of the command line. This prevents errors by the users (such as attempts to use wrong CRI during upgrade).
@@ -226,9 +228,9 @@ func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, w
 	fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", internalcfg.KubernetesVersion)
 
 	if flags.dryRun {
-		return upgrade.DryRunStaticPodUpgrade(internalcfg)
+		return upgrade.DryRunStaticPodUpgrade(flags.kustomizeDir, internalcfg)
 	}
 
 	// Don't save etcd backup directory if etcd is HA, as this could cause corruption
-	return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts)
+	return upgrade.PerformStaticPodUpgrade(client, waiter, internalcfg, flags.etcdUpgrade, flags.renewCerts, flags.kustomizeDir)
 }

cmd/kubeadm/app/cmd/upgrade/node.go

@@ -45,6 +45,7 @@ type nodeOptions struct {
 	etcdUpgrade      bool
 	renewCerts       bool
 	dryRun           bool
+	kustomizeDir     string
 }
 
 // compile-time assert that the local data object satisfies the phases data interface.
@@ -60,6 +61,7 @@ type nodeData struct {
 	cfg                *kubeadmapi.InitConfiguration
 	isControlPlaneNode bool
 	client             clientset.Interface
+	kustomizeDir       string
 }
 
 // NewCmdNode returns the cobra command for `kubeadm upgrade node`
@@ -80,6 +82,7 @@ func NewCmdNode() *cobra.Command {
 	// adds flags to the node command
 	// flags could be eventually inherited by the sub-commands automatically generated for phases
 	addUpgradeNodeFlags(cmd.Flags(), nodeOptions)
+	options.AddKustomizePodsFlag(cmd.Flags(), &nodeOptions.kustomizeDir)
 
 	// initialize the workflow runner with the list of phases
 	nodeRunner.AppendPhase(phases.NewControlPlane())
@@ -151,6 +154,7 @@ func newNodeData(cmd *cobra.Command, args []string, options *nodeOptions) (*node
 		cfg:                cfg,
 		client:             client,
 		isControlPlaneNode: isControlPlaneNode,
+		kustomizeDir:       options.kustomizeDir,
 	}, nil
 }
 
@@ -189,6 +193,11 @@ func (d *nodeData) Client() clientset.Interface {
 	return d.client
 }
 
+// KustomizeDir returns the folder where kustomize patches for static pod manifest are stored
+func (d *nodeData) KustomizeDir() string {
+	return d.kustomizeDir
+}
+
 // NewCmdUpgradeNodeConfig returns the cobra.Command for downloading the new/upgrading the kubelet configuration from the kubelet-config-1.X
 // ConfigMap in the cluster
 // TODO: to remove when 1.18 is released

cmd/kubeadm/app/phases/etcd/local.go

@@ -122,7 +122,7 @@ func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadm
 // CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
 // for an additional etcd member that is joining an existing local/stacked etcd cluster.
 // Other members of the etcd cluster will be notified of the joining node in beforehand as well.
-func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
+func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, kustomizeDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
 	// creates an etcd client that connects to all the local/stacked etcd members
 	klog.V(1).Info("creating etcd client that connects to etcd pods")
 	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
@@ -141,16 +141,25 @@ func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifest
 	fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster")
 	klog.V(1).Infof("Updated etcd member list: %v", initialCluster)
 
-	klog.V(1).Info("Creating local etcd static pod manifest file")
+	fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd)
+
 	// gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members
 	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)
+
+	// if kustomizeDir is defined, customize the static pod manifest
+	if kustomizeDir != "" {
+		kustomizedSpec, err := staticpodutil.KustomizeStaticPod(&spec, kustomizeDir)
+		if err != nil {
+			return errors.Wrapf(err, "failed to kustomize static pod manifest file for %q", kubeadmconstants.Etcd)
+		}
+		spec = *kustomizedSpec
+	}
+
 	// writes etcd StaticPod to disk
 	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
 		return err
 	}
 
-	fmt.Printf("[etcd] Wrote Static Pod manifest for a local etcd member to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir))
-
 	fmt.Printf("[etcd] Waiting for the new etcd member to join the cluster. This can take up to %v\n", etcdHealthyCheckInterval*etcdHealthyCheckRetries)
 	if _, err := etcdClient.WaitForClusterAvailable(etcdHealthyCheckRetries, etcdHealthyCheckInterval); err != nil {
 		return err