Fixes /kubernetes/kubeadm/issues/1072 Cleanup of selfhosting logic

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -603,11 +603,10 @@ func TestValidateFeatureGates(t *testing.T) {
 		featureGates featureFlag
 		expected     bool
 	}{
-		{featureFlag{"SelfHosting": true}, true},
-		{featureFlag{"SelfHosting": false}, true},
-		{featureFlag{"StoreCertsInSecrets": true}, true},
-		{featureFlag{"StoreCertsInSecrets": false}, true},
-		{featureFlag{"Foo": true}, false},
+		{featureFlag{"Unknown": true}, false},
+		{featureFlag{"Unknown": false}, false},
+		{featureFlag{"CoreDNS": true}, true},
+		{featureFlag{"CoreDNS": false}, true},
 	}
 	for _, rt := range tests {
 		actual := ValidateFeatureGates(rt.featureGates, nil)

cmd/kubeadm/app/cmd/BUILD

@@ -47,7 +47,6 @@ go_library(
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",
         "//cmd/kubeadm/app/phases/markmaster:go_default_library",
         "//cmd/kubeadm/app/phases/patchnode:go_default_library",
-        "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
         "//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",

cmd/kubeadm/app/cmd/alpha/BUILD

@@ -8,6 +8,7 @@ go_library(
         "kubeconfig.go",
         "kubelet.go",
         "preflight.go",
+        "selfhosting.go",
     ],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/cmd/alpha",
     visibility = ["//visibility:public"],
@@ -19,12 +20,15 @@ go_library(
         "//cmd/kubeadm/app/cmd/phases:go_default_library",
         "//cmd/kubeadm/app/cmd/util:go_default_library",
         "//cmd/kubeadm/app/constants:go_default_library",
+        "//cmd/kubeadm/app/features:go_default_library",
         "//cmd/kubeadm/app/phases/certs:go_default_library",
         "//cmd/kubeadm/app/phases/certs/renewal:go_default_library",
         "//cmd/kubeadm/app/phases/kubeconfig:go_default_library",
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",
+        "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/util/normalizer:go_default_library",

cmd/kubeadm/app/cmd/alpha/alpha.go

@@ -25,7 +25,7 @@ import (
 )
 
 // NewCmdAlpha returns "kubeadm alpha" command.
-func NewCmdAlpha(out io.Writer) *cobra.Command {
+func NewCmdAlpha(in io.Reader, out io.Writer) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "alpha",
 		Short: "Kubeadm experimental sub-commands",
@@ -35,6 +35,7 @@ func NewCmdAlpha(out io.Writer) *cobra.Command {
 	cmd.AddCommand(newCmdKubeletUtility())
 	cmd.AddCommand(newCmdKubeConfigUtility(out))
 	cmd.AddCommand(newCmdPreFlightUtility())
+	cmd.AddCommand(NewCmdSelfhosting(in))
 
 	// TODO: This command should be removed as soon as the kubeadm init phase refactoring is completed.
 	//		 current phases implemented as cobra.Commands should become workflow.Phases, while other utilities
@@ -54,7 +55,6 @@ func newCmdPhase(out io.Writer) *cobra.Command {
 	cmd.AddCommand(phases.NewCmdAddon())
 	cmd.AddCommand(phases.NewCmdBootstrapToken())
 	cmd.AddCommand(phases.NewCmdMarkMaster())
-	cmd.AddCommand(phases.NewCmdSelfhosting())
 
 	return cmd
 }

cmd/kubeadm/app/cmd/phases/selfhosting.go → cmd/kubeadm/app/cmd/alpha/selfhosting.go

@@ -14,19 +14,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package phases
+package alpha
 
 import (
-	"os"
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
 	"strings"
-	"time"
 
 	"github.com/spf13/cobra"
 
 	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
 	kubeadmapiv1beta1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
+	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/phases"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
@@ -36,6 +39,9 @@ import (
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/pkg/util/normalizer"
+
+	"os"
+	"time"
 )
 
 var (
@@ -47,46 +53,63 @@ var (
 		` + cmdutil.AlphaDisclaimer)
 
 	selfhostingExample = normalizer.Examples(`
-		# Converts a static Pod-hosted control plane into a self-hosted one, 
-		# functionally equivalent to what generated by kubeadm init executed
-		# with --feature-gates=SelfHosting=true. 
+		# Converts a static Pod-hosted control plane into a self-hosted one. 
 
-		kubeadm alpha phase selfhosting convert-from-staticpods
+		kubeadm alpha phase self-hosting convert-from-staticpods
 		`)
 )
 
 // NewCmdSelfhosting returns the self-hosting Cobra command
-func NewCmdSelfhosting() *cobra.Command {
+func NewCmdSelfhosting(in io.Reader) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "selfhosting",
 		Aliases: []string{"selfhosted", "self-hosting"},
 		Short:   "Makes a kubeadm cluster self-hosted",
 		Long:    cmdutil.MacroCommandLongDescription,
 	}
 
-	cmd.AddCommand(getSelfhostingSubCommand())
+	cmd.AddCommand(getSelfhostingSubCommand(in))
 	return cmd
 }
 
-// getSelfhostingSubCommand returns sub commands for Selfhosting phase
-func getSelfhostingSubCommand() *cobra.Command {
+// getSelfhostingSubCommand returns sub commands for Self-hosting phase
+func getSelfhostingSubCommand(in io.Reader) *cobra.Command {
 
 	cfg := &kubeadmapiv1beta1.InitConfiguration{}
 	// Default values for the cobra help text
 	kubeadmscheme.Scheme.Default(cfg)
 
 	var cfgPath, featureGatesString string
+	forcePivot, certsInSecrets := false, false
 	kubeConfigFile := constants.GetAdminKubeConfigPath()
 
 	// Creates the UX Command
 	cmd := &cobra.Command{
-		Use:     "convert-from-staticpods",
+		Use:     "pivot",
 		Aliases: []string{"from-staticpods"},
 		Short:   "Converts a static Pod-hosted control plane into a self-hosted one",
 		Long:    selfhostingLongDesc,
 		Example: selfhostingExample,
 		Run: func(cmd *cobra.Command, args []string) {
+
 			var err error
+
+			if !forcePivot {
+				fmt.Println("WARNING: self-hosted clusters are not supported by kubeadm upgrade and by other kubeadm commands!")
+				fmt.Print("[pivot] are you sure you want to proceed? [y/n]: ")
+				s := bufio.NewScanner(in)
+				s.Scan()
+
+				err = s.Err()
+				kubeadmutil.CheckErr(err)
+
+				if strings.ToLower(s.Text()) != "y" {
+					kubeadmutil.CheckErr(errors.New("aborted pivot operation"))
+				}
+			}
+
+			fmt.Println("[pivot] pivoting cluster to self-hosted")
+
 			if cfg.FeatureGates, err = features.NewFeatureGate(&features.InitFeatureGates, featureGatesString); err != nil {
 				kubeadmutil.CheckErr(err)
 			}
@@ -102,15 +125,15 @@ func getSelfhostingSubCommand() *cobra.Command {
 
 			// KubernetesVersion is not used, but we set it explicitly to avoid the lookup
 			// of the version from the internet when executing ConfigFileAndDefaultsToInternalConfig
-			SetKubernetesVersion(cfg)
+			phases.SetKubernetesVersion(cfg)
 
 			// This call returns the ready-to-use configuration based on the configuration file that might or might not exist and the default cfg populated by flags
 			internalcfg, err := configutil.ConfigFileAndDefaultsToInternalConfig(cfgPath, cfg)
 			kubeadmutil.CheckErr(err)
 
 			// Converts the Static Pod-hosted control plane into a self-hosted one
 			waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout)
-			err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false)
+			err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter, false, certsInSecrets)
 			kubeadmutil.CheckErr(err)
 		},
 	}
@@ -119,8 +142,15 @@ func getSelfhostingSubCommand() *cobra.Command {
 	// flags bound to the configuration object
 	cmd.Flags().StringVar(&cfg.CertificatesDir, "cert-dir", cfg.CertificatesDir, `The path where certificates are stored`)
 	cmd.Flags().StringVar(&cfgPath, "config", cfgPath, "Path to a kubeadm config file. WARNING: Usage of a configuration file is experimental")
-	cmd.Flags().StringVar(&featureGatesString, "feature-gates", featureGatesString, "A set of key=value pairs that describe feature gates for various features. "+
-		"Options are:\n"+strings.Join(features.KnownFeatures(&features.InitFeatureGates), "\n"))
+
+	cmd.Flags().BoolVarP(
+		&certsInSecrets, "store-certs-in-secrets", "s",
+		false, "Enable storing certs in secrets")
+
+	cmd.Flags().BoolVarP(
+		&forcePivot, "force", "f", false,
+		"Pivot the cluster without prompting for confirmation",
+	)
 
 	// flags that are not bound to the configuration object
 	// Note: All flags that are not bound to the cfg object should be whitelisted in cmd/kubeadm/app/apis/kubeadm/validation/validation.go

cmd/kubeadm/app/cmd/cmd.go

@@ -89,7 +89,7 @@ func NewKubeadmCommand(in io.Reader, out, err io.Writer) *cobra.Command {
 	cmds.AddCommand(NewCmdVersion(out))
 	cmds.AddCommand(NewCmdToken(out, err))
 	cmds.AddCommand(upgrade.NewCmdUpgrade(out))
-	cmds.AddCommand(alpha.NewCmdAlpha(out))
+	cmds.AddCommand(alpha.NewCmdAlpha(in, out))
 
 	AddKubeadmOtherFlags(cmds.PersistentFlags(), &rootfsPath)
 

cmd/kubeadm/app/cmd/init.go

@@ -50,7 +50,7 @@ import (
 	kubeconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubeconfig"
 	kubeletphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/kubelet"
 	markmasterphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/markmaster"
-	selfhostingphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
+	uploadconfigphase "k8s.io/kubernetes/cmd/kubeadm/app/phases/uploadconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/preflight"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
@@ -462,7 +462,7 @@ func runInit(i *initData, out io.Writer) error {
 
 	// Get directories to write files to; can be faked if we're dry-running
 	glog.V(1).Infof("[init] Getting certificates directory from configuration")
-	certsDirToWriteTo, kubeConfigDir, manifestDir, _, err := getDirectoriesToUse(i.dryRun, i.dryRunDir, i.cfg.CertificatesDir)
+	certsDirToWriteTo, kubeConfigDir, _, _, err := getDirectoriesToUse(i.dryRun, i.dryRunDir, i.cfg.CertificatesDir)
 	if err != nil {
 		return errors.Wrap(err, "error getting directories to use")
 	}
@@ -479,11 +479,17 @@ func runInit(i *initData, out io.Writer) error {
 		return errors.Wrap(err, "failed to create client")
 	}
 
-	// TODO: NewControlPlaneWaiter should be converted to private after the self-hosting phase is removed.
-	timeout := i.cfg.ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration
-	waiter, err := phases.NewControlPlaneWaiter(i.dryRun, timeout, client, i.outputWriter)
-	if err != nil {
-		return errors.Wrap(err, "failed to create waiter")
+	// Upload currently used configuration to the cluster
+	// Note: This is done right in the beginning of cluster initialization; as we might want to make other phases
+	// depend on centralized information from this source in the future
+	glog.V(1).Infof("[init] uploading currently used configuration to the cluster")
+	if err := uploadconfigphase.UploadConfiguration(i.cfg, client); err != nil {
+		return errors.Wrap(err, "error uploading configuration")
+	}
+
+	glog.V(1).Infof("[init] creating kubelet configuration configmap")
+	if err := kubeletphase.CreateConfigMap(i.cfg, client); err != nil {
+		return errors.Wrap(err, "error creating kubelet configuration ConfigMap")
 	}
 
 	// PHASE 4: Mark the master with the right label/taint
@@ -560,17 +566,6 @@ func runInit(i *initData, out io.Writer) error {
 		return errors.Wrap(err, "error ensuring proxy addon")
 	}
 
-	// PHASE 7: Make the control plane self-hosted if feature gate is enabled
-	if features.Enabled(i.cfg.FeatureGates, features.SelfHosting) {
-		glog.V(1).Infof("[init] feature gate is enabled. Making control plane self-hosted")
-		// Temporary control plane is up, now we create our self hosted control
-		// plane components and remove the static manifests:
-		fmt.Println("[self-hosted] creating self-hosted control plane")
-		if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter, i.dryRun); err != nil {
-			return errors.Wrap(err, "error creating self hosted control plane")
-		}
-	}
-
 	// Exit earlier if we're dryrunning
 	if i.dryRun {
 		fmt.Println("[dryrun] finished dry-running successfully. Above are the resources that would be created")

cmd/kubeadm/app/cmd/join.go

@@ -410,16 +410,6 @@ func (j *Join) CheckIfReadyForAdditionalControlPlane(initConfiguration *kubeadma
 		return errors.New("unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address")
 	}
 
-	// blocks if control plane is self-hosted
-	if features.Enabled(initConfiguration.FeatureGates, features.SelfHosting) {
-		return errors.New("self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`")
-	}
-
-	// blocks if the certificates for the control plane are stored in secrets (instead of the local pki folder)
-	if features.Enabled(initConfiguration.FeatureGates, features.StoreCertsInSecrets) {
-		return errors.New("certificates stored in secrets, as well as self-hosted clusters are deprecated and won't be supported by `kubeadm join --experimental-control-plane`")
-	}
-
 	// checks if the certificates that must be equal across contolplane instances are provided
 	if ret, err := certsphase.SharedCertificateExists(initConfiguration); !ret {
 		return err

cmd/kubeadm/app/cmd/phases/BUILD

@@ -12,7 +12,6 @@ go_library(
         "kubelet.go",
         "markmaster.go",
         "preflight.go",
-        "selfhosting.go",
         "uploadconfig.go",
         "util.go",
         "waitcontrolplane.go",
@@ -40,7 +39,6 @@ go_library(
         "//cmd/kubeadm/app/phases/kubelet:go_default_library",
         "//cmd/kubeadm/app/phases/markmaster:go_default_library",
         "//cmd/kubeadm/app/phases/patchnode:go_default_library",
-        "//cmd/kubeadm/app/phases/selfhosting:go_default_library",
         "//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",

cmd/kubeadm/app/cmd/phases/addons.go

@@ -49,7 +49,7 @@ var (
 		# Installs the CoreDNS and the kube-proxy addons components via the API server,
 		# functionally equivalent to what installed by kubeadm init. 
 
-		kubeadm alpha phase selfhosting from-staticpods
+		kubeadm alpha phase self-hosting from-staticpods
 		`)
 
 	corednsAddonsLongDesc = normalizer.LongDesc(`

cmd/kubeadm/app/cmd/phases/waitcontrolplane.go

@@ -94,7 +94,7 @@ func runWaitControlPlanePhase(c workflow.RunData) error {
 	}
 
 	timeout := data.Cfg().ClusterConfiguration.APIServer.TimeoutForControlPlane.Duration
-	waiter, err := NewControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter())
+	waiter, err := newControlPlaneWaiter(data.DryRun(), timeout, client, data.OutputWriter())
 	if err != nil {
 		return errors.Wrap(err, "error creating waiter")
 	}
@@ -144,7 +144,7 @@ func printFilesIfDryRunning(data waitControlPlaneData) error {
 
 // NewControlPlaneWaiter returns a new waiter that is used to wait on the control plane to boot up.
 // TODO: make private (lowercase) after self-hosting phase is removed.
-func NewControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) {
+func newControlPlaneWaiter(dryRun bool, timeout time.Duration, client clientset.Interface, out io.Writer) (apiclient.Waiter, error) {
 	if dryRun {
 		return dryrunutil.NewWaiter(), nil
 	}

cmd/kubeadm/app/cmd/upgrade/apply.go

@@ -270,16 +270,6 @@ func EnforceVersionPolicies(flags *applyFlags, versionGetter upgrade.VersionGett
 // PerformControlPlaneUpgrade actually performs the upgrade procedure for the cluster of your type (self-hosted or static-pod-hosted)
 func PerformControlPlaneUpgrade(flags *applyFlags, client clientset.Interface, waiter apiclient.Waiter, internalcfg *kubeadmapi.InitConfiguration) error {
 
-	// Check if the cluster is self-hosted and act accordingly
-	glog.V(1).Infoln("checking if cluster is self-hosted")
-	if upgrade.IsControlPlaneSelfHosted(client) {
-		fmt.Printf("[upgrade/apply] Upgrading your Self-Hosted control plane to version %q...\n", flags.newK8sVersionStr)
-
-		// Upgrade the self-hosted cluster
-		glog.V(1).Infoln("[upgrade/apply] upgrading self-hosted cluster")
-		return upgrade.SelfHostedControlPlane(client, waiter, internalcfg, flags.newK8sVersion)
-	}
-
 	// OK, the cluster is hosted using static pods. Upgrade a static-pod hosted cluster
 	fmt.Printf("[upgrade/apply] Upgrading your Static Pod-hosted control plane to version %q...\n", flags.newK8sVersionStr)
 

cmd/kubeadm/app/cmd/upgrade/common.go

@@ -58,6 +58,11 @@ func enforceRequirements(flags *applyPlanFlags, dryRun bool, newK8sVersion strin
 		return nil, errors.Wrapf(err, "couldn't create a Kubernetes client from file %q", flags.kubeConfigPath)
 	}
 
+	// Check if the cluster is self-hosted
+	if upgrade.IsControlPlaneSelfHosted(client) {
+		return nil, errors.Errorf("cannot upgrade a self-hosted control plane")
+	}
+
 	// Run healthchecks against the cluster
 	if err := upgrade.CheckClusterHealth(client, flags.ignorePreflightErrorsSet); err != nil {
 		return nil, errors.Wrap(err, "[upgrade/health] FATAL")