kubeadm: Fully implement 'kubeadm init --dry-run'

cmd/kubeadm/app/cmd/BUILD

@@ -45,6 +45,7 @@ go_library(
         "//cmd/kubeadm/app/util:go_default_library",
         "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
+        "//cmd/kubeadm/app/util/dryrun:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//cmd/kubeadm/app/util/pubkeypin:go_default_library",
         "//cmd/kubeadm/app/util/token:go_default_library",

cmd/kubeadm/app/cmd/config.go

@@ -139,6 +139,8 @@ func NewCmdConfigUploadFromFlags(out io.Writer, kubeConfigFile *string) *cobra.C
 			Using from-flags, you can upload configuration to the ConfigMap in the cluster using the same flags you'd give to kubeadm init.
 			If you initialized your cluster using a v1.7.x or lower kubeadm client and set some flag; you need to run this command with the
 			same flags before upgrading to v1.8 using 'kubeadm upgrade'.
+
+			The configuration is located in the %q namespace in the %q ConfigMap
 		`), metav1.NamespaceSystem, constants.MasterConfigurationConfigMap),
 		Run: func(cmd *cobra.Command, args []string) {
 			var err error

cmd/kubeadm/app/cmd/init.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/template"
 	"time"
@@ -53,6 +54,7 @@ import (
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
+	dryrunutil "k8s.io/kubernetes/cmd/kubeadm/app/util/dryrun"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/cmd/kubeadm/app/util/pubkeypin"
 	"k8s.io/kubernetes/pkg/api"
@@ -264,33 +266,67 @@ func (i *Init) Run(out io.Writer) error {
 		return fmt.Errorf("couldn't parse kubernetes version %q: %v", i.cfg.KubernetesVersion, err)
 	}
 
+	// Get directories to write files to; can be faked if we're dry-running
+	realCertsDir := i.cfg.CertificatesDir
+	certsDirToWriteTo, kubeConfigDir, manifestDir, err := getDirectoriesToUse(i.dryRun, i.cfg.CertificatesDir)
+	if err != nil {
+		return err
+	}
+	// certsDirToWriteTo is gonna equal cfg.CertificatesDir in the normal case, but gonna be a temp directory if dryrunning
+	i.cfg.CertificatesDir = certsDirToWriteTo
+
+	adminKubeConfigPath := filepath.Join(kubeConfigDir, kubeadmconstants.AdminKubeConfigFileName)
+
 	// PHASE 1: Generate certificates
 	if err := certsphase.CreatePKIAssets(i.cfg); err != nil {
 		return err
 	}
 
 	// PHASE 2: Generate kubeconfig files for the admin and the kubelet
-	if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeadmconstants.KubernetesDir, i.cfg); err != nil {
+	if err := kubeconfigphase.CreateInitKubeConfigFiles(kubeConfigDir, i.cfg); err != nil {
 		return err
 	}
 
+	// Temporarily set cfg.CertificatesDir to the "real value" when writing controlplane manifests
+	// This is needed for writing the right kind of manifests
+	i.cfg.CertificatesDir = realCertsDir
+
 	// PHASE 3: Bootstrap the control plane
-	manifestPath := kubeadmconstants.GetStaticPodDirectory()
-	if err := controlplanephase.CreateInitStaticPodManifestFiles(manifestPath, i.cfg); err != nil {
+	if err := controlplanephase.CreateInitStaticPodManifestFiles(manifestDir, i.cfg); err != nil {
 		return err
 	}
 	// Add etcd static pod spec only if external etcd is not configured
 	if len(i.cfg.Etcd.Endpoints) == 0 {
-		if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(manifestPath, i.cfg); err != nil {
+		if err := etcdphase.CreateLocalEtcdStaticPodManifestFile(manifestDir, i.cfg); err != nil {
 			return err
 		}
 	}
 
-	client, err := createClientsetAndOptionallyWaitForReady(i.cfg, i.dryRun)
+	// Revert the earlier CertificatesDir assignment to the directory that can be written to
+	i.cfg.CertificatesDir = certsDirToWriteTo
+
+	// If we're dry-running, print the generated manifests
+	if err := printFilesIfDryRunning(i.dryRun, manifestDir); err != nil {
+		return err
+	}
+
+	// Create a kubernetes client and wait for the API server to be healthy (if not dryrunning)
+	client, err := createClient(i.cfg, i.dryRun)
 	if err != nil {
 		return err
 	}
 
+	// waiter holds the apiclient.Waiter implementation of choice, responsible for querying the API server in various ways and waiting for conditions to be fulfilled
+	waiter := getWaiter(i.dryRun, client)
+
+	fmt.Printf("[init] Waiting for the kubelet to boot up the control plane as Static Pods from directory %q\n", kubeadmconstants.GetStaticPodDirectory())
+	fmt.Println("[init] This process often takes about a minute to perform or longer if the control plane images have to be pulled...")
+	// TODO: Adjust this timeout or start polling the kubelet API
+	// TODO: Make this timeout more realistic when we do create some more complex logic about the interaction with the kubelet
+	if err := waiter.WaitForAPI(); err != nil {
+		return err
+	}
+
 	// PHASE 4: Mark the master with the right label/taint
 	if err := markmasterphase.MarkMaster(client, i.cfg.NodeName); err != nil {
 		return err
@@ -316,7 +352,7 @@ func (i *Init) Run(out io.Writer) error {
 	}
 
 	// Create the cluster-info ConfigMap with the associated RBAC rules
-	if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, kubeadmconstants.GetAdminKubeConfigPath()); err != nil {
+	if err := clusterinfophase.CreateBootstrapConfigMapIfNotExists(client, adminKubeConfigPath); err != nil {
 		return err
 	}
 	if err := clusterinfophase.CreateClusterInfoRBACRules(client); err != nil {
@@ -347,11 +383,17 @@ func (i *Init) Run(out io.Writer) error {
 		// Temporary control plane is up, now we create our self hosted control
 		// plane components and remove the static manifests:
 		fmt.Println("[self-hosted] Creating self-hosted control plane...")
-		if err := selfhostingphase.CreateSelfHostedControlPlane(i.cfg, client); err != nil {
+		if err := selfhostingphase.CreateSelfHostedControlPlane(manifestDir, kubeConfigDir, i.cfg, client, waiter); err != nil {
 			return err
 		}
 	}
 
+	// Exit earlier if we're dryrunning
+	if i.dryRun {
+		fmt.Println("[dryrun] Finished dry-running successfully; above are the resources that would be created.")
+		return nil
+	}
+
 	// Load the CA certificate from so we can pin its public key
 	caCert, err := pkiutil.TryLoadCertFromDisk(i.cfg.CertificatesDir, kubeadmconstants.CACertAndKeyBaseName)
 
@@ -362,8 +404,7 @@ func (i *Init) Run(out io.Writer) error {
 	}
 
 	ctx := map[string]string{
-		"KubeConfigPath": kubeadmconstants.GetAdminKubeConfigPath(),
-		"KubeConfigName": kubeadmconstants.AdminKubeConfigFileName,
+		"KubeConfigPath": adminKubeConfigPath,
 		"Token":          i.cfg.Token,
 		"CAPubKeyPin":    pubkeypin.Hash(caCert),
 		"MasterHostPort": masterHostPort,
@@ -375,24 +416,59 @@ func (i *Init) Run(out io.Writer) error {
 	return initDoneTempl.Execute(out, ctx)
 }
 
-func createClientsetAndOptionallyWaitForReady(cfg *kubeadmapi.MasterConfiguration, dryRun bool) (clientset.Interface, error) {
+// createClient creates a clientset.Interface object
+func createClient(cfg *kubeadmapi.MasterConfiguration, dryRun bool) (clientset.Interface, error) {
 	if dryRun {
 		// If we're dry-running; we should create a faked client that answers some GETs in order to be able to do the full init flow and just logs the rest of requests
 		dryRunGetter := apiclient.NewInitDryRunGetter(cfg.NodeName, cfg.Networking.ServiceSubnet)
 		return apiclient.NewDryRunClient(dryRunGetter, os.Stdout), nil
 	}
 
-	// If we're acting for real,we should create a connection to the API server and wait for it to come up
-	client, err := kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath())
-	if err != nil {
-		return nil, err
+	// If we're acting for real, we should create a connection to the API server and wait for it to come up
+	return kubeconfigutil.ClientSetFromFile(kubeadmconstants.GetAdminKubeConfigPath())
+}
+
+// getDirectoriesToUse returns the (in order) certificates, kubeconfig and Static Pod manifest directories, followed by a possible error
+// This behaves differently when dry-running vs the normal flow
+func getDirectoriesToUse(dryRun bool, defaultPkiDir string) (string, string, string, error) {
+	if dryRun {
+		dryRunDir, err := ioutil.TempDir("", "kubeadm-init-dryrun")
+		if err != nil {
+			return "", "", "", fmt.Errorf("couldn't create a temporary directory: %v", err)
+		}
+		// Use the same temp dir for all
+		return dryRunDir, dryRunDir, dryRunDir, nil
 	}
 
-	fmt.Printf("[init] Waiting for the kubelet to boot up the control plane as Static Pods from directory %q\n", kubeadmconstants.GetStaticPodDirectory())
-	// TODO: Adjust this timeout or start polling the kubelet API
-	// TODO: Make this timeout more realistic when we do create some more complex logic about the interaction with the kubelet
-	if err := apiclient.WaitForAPI(client, 30*time.Minute); err != nil {
-		return nil, err
+	return defaultPkiDir, kubeadmconstants.KubernetesDir, kubeadmconstants.GetStaticPodDirectory(), nil
+}
+
+// printFilesIfDryRunning prints the Static Pod manifests to stdout and informs about the temporary directory to go and lookup
+func printFilesIfDryRunning(dryRun bool, manifestDir string) error {
+	if !dryRun {
+		return nil
+	}
+
+	fmt.Printf("[dryrun] Wrote certificates, kubeconfig files and control plane manifests to %q\n", manifestDir)
+	fmt.Println("[dryrun] Won't print certificates or kubeconfig files due to the sensitive nature of them")
+	fmt.Printf("[dryrun] Please go and examine the %q directory for details about what would be written\n", manifestDir)
+
+	// Print the contents of the upgraded manifests and pretend like they were in /etc/kubernetes/manifests
+	files := []dryrunutil.FileToPrint{}
+	for _, component := range kubeadmconstants.MasterComponents {
+		realPath := kubeadmconstants.GetStaticPodFilepath(component, manifestDir)
+		outputPath := kubeadmconstants.GetStaticPodFilepath(component, kubeadmconstants.GetStaticPodDirectory())
+		files = append(files, dryrunutil.NewFileToPrint(realPath, outputPath))
+	}
+
+	return dryrunutil.PrintDryRunFiles(files, os.Stdout)
+}
+
+// getWaiter gets the right waiter implementation
+func getWaiter(dryRun bool, client clientset.Interface) apiclient.Waiter {
+	if dryRun {
+		return dryrunutil.NewWaiter()
 	}
-	return client, nil
+	// TODO: Adjust this timeout slightly?
+	return apiclient.NewKubeWaiter(client, 30*time.Minute, os.Stdout)
 }

cmd/kubeadm/app/cmd/phases/BUILD

@@ -39,6 +39,7 @@ go_library(
         "//cmd/kubeadm/app/phases/uploadconfig:go_default_library",
         "//cmd/kubeadm/app/preflight:go_default_library",
         "//cmd/kubeadm/app/util:go_default_library",
+        "//cmd/kubeadm/app/util/apiclient:go_default_library",
         "//cmd/kubeadm/app/util/config:go_default_library",
         "//cmd/kubeadm/app/util/kubeconfig:go_default_library",
         "//pkg/api:go_default_library",

cmd/kubeadm/app/cmd/phases/selfhosting.go

@@ -17,16 +17,20 @@ limitations under the License.
 package phases
 
 import (
+	"os"
 	"strings"
+	"time"
 
 	"github.com/spf13/cobra"
 
 	kubeadmapiext "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1alpha1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/validation"
 	cmdutil "k8s.io/kubernetes/cmd/kubeadm/app/cmd/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/kubernetes/cmd/kubeadm/app/phases/selfhosting"
 	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
+	"k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient"
 	configutil "k8s.io/kubernetes/cmd/kubeadm/app/util/config"
 	kubeconfigutil "k8s.io/kubernetes/cmd/kubeadm/app/util/kubeconfig"
 	"k8s.io/kubernetes/pkg/api"
@@ -78,7 +82,8 @@ func getSelfhostingSubCommand() *cobra.Command {
 			kubeadmutil.CheckErr(err)
 
 			// Converts the Static Pod-hosted control plane into a self-hosted one
-			err = selfhosting.CreateSelfHostedControlPlane(internalcfg, client)
+			waiter := apiclient.NewKubeWaiter(client, 2*time.Minute, os.Stdout)
+			err = selfhosting.CreateSelfHostedControlPlane(constants.GetStaticPodDirectory(), constants.KubernetesDir, internalcfg, client, waiter)
 			kubeadmutil.CheckErr(err)
 		},
 	}

cmd/kubeadm/app/phases/selfhosting/selfhosting.go

@@ -54,7 +54,10 @@ const (
 // 8. In order to avoid race conditions, we have to make sure that static pod is deleted correctly before we continue
 //      Otherwise, there is a race condition when we proceed without kubelet having restarted the API server correctly and the next .Create call flakes
 // 9. Do that for the kube-apiserver, kube-controller-manager and kube-scheduler in a loop
-func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client clientset.Interface) error {
+func CreateSelfHostedControlPlane(manifestsDir, kubeConfigDir string, cfg *kubeadmapi.MasterConfiguration, client clientset.Interface, waiter apiclient.Waiter) error {
+
+	// Adjust the timeout slightly to something self-hosting specific
+	waiter.SetTimeout(selfHostingWaitTimeout)
 
 	// Here the map of different mutators to use for the control plane's podspec is stored
 	mutators := getDefaultMutators()
@@ -66,7 +69,7 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client cl
 		if err := uploadTLSSecrets(client, cfg.CertificatesDir); err != nil {
 			return err
 		}
-		if err := uploadKubeConfigSecrets(client); err != nil {
+		if err := uploadKubeConfigSecrets(client, kubeConfigDir); err != nil {
 			return err
 		}
 		// Add the store-certs-in-secrets-specific mutators here so that the self-hosted component starts using them
@@ -77,7 +80,7 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client cl
 
 	for _, componentName := range kubeadmconstants.MasterComponents {
 		start := time.Now()
-		manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, kubeadmconstants.GetStaticPodDirectory())
+		manifestPath := kubeadmconstants.GetStaticPodFilepath(componentName, manifestsDir)
 
 		// Since we want this function to be idempotent; just continue and try the next component if this file doesn't exist
 		if _, err := os.Stat(manifestPath); err != nil {
@@ -102,7 +105,7 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client cl
 		}
 
 		// Wait for the self-hosted component to come up
-		if err := apiclient.WaitForPodsWithLabel(client, selfHostingWaitTimeout, os.Stdout, buildSelfHostedWorkloadLabelQuery(componentName)); err != nil {
+		if err := waiter.WaitForPodsWithLabel(buildSelfHostedWorkloadLabelQuery(componentName)); err != nil {
 			return err
 		}
 
@@ -115,12 +118,12 @@ func CreateSelfHostedControlPlane(cfg *kubeadmapi.MasterConfiguration, client cl
 		// remove the Static Pod (or the mirror Pod respectively). This implicitely also tests that the API server endpoint is healthy,
 		// because this blocks until the API server returns a 404 Not Found when getting the Static Pod
 		staticPodName := fmt.Sprintf("%s-%s", componentName, cfg.NodeName)
-		if err := apiclient.WaitForStaticPodToDisappear(client, selfHostingWaitTimeout, staticPodName); err != nil {
+		if err := waiter.WaitForPodToDisappear(staticPodName); err != nil {
 			return err
 		}
 
 		// Just as an extra safety check; make sure the API server is returning ok at the /healthz endpoint (although we know it could return a GET answer for a Pod above)
-		if err := apiclient.WaitForAPI(client, selfHostingWaitTimeout); err != nil {
+		if err := waiter.WaitForAPI(); err != nil {
 			return err
 		}
 

cmd/kubeadm/app/phases/selfhosting/selfhosting_volumes.go

@@ -19,7 +19,7 @@ package selfhosting
 import (
 	"fmt"
 	"io/ioutil"
-	"path"
+	"path/filepath"
 
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -184,8 +184,8 @@ func uploadTLSSecrets(client clientset.Interface, certDir string) error {
 	for _, tlsKeyPair := range getTLSKeyPairs() {
 		secret, err := createTLSSecretFromFiles(
 			tlsKeyPair.name,
-			path.Join(certDir, tlsKeyPair.cert),
-			path.Join(certDir, tlsKeyPair.key),
+			filepath.Join(certDir, tlsKeyPair.cert),
+			filepath.Join(certDir, tlsKeyPair.key),
 		)
 		if err != nil {
 			return err
@@ -200,13 +200,13 @@ func uploadTLSSecrets(client clientset.Interface, certDir string) error {
 	return nil
 }
 
-func uploadKubeConfigSecrets(client clientset.Interface) error {
+func uploadKubeConfigSecrets(client clientset.Interface, kubeConfigDir string) error {
 	files := []string{
 		kubeadmconstants.SchedulerKubeConfigFileName,
 		kubeadmconstants.ControllerManagerKubeConfigFileName,
 	}
 	for _, file := range files {
-		kubeConfigPath := path.Join(kubeadmconstants.KubernetesDir, file)
+		kubeConfigPath := filepath.Join(kubeConfigDir, file)
 		secret, err := createOpaqueSecretFromFile(file, kubeConfigPath)
 		if err != nil {
 			return err
@@ -257,7 +257,7 @@ func createOpaqueSecretFromFile(secretName, file string) (*v1.Secret, error) {
 		},
 		Type: v1.SecretTypeOpaque,
 		Data: map[string][]byte{
-			path.Base(file): fileBytes,
+			filepath.Base(file): fileBytes,
 		},
 	}, nil
 }

cmd/kubeadm/app/util/BUILD

@@ -51,6 +51,7 @@ filegroup(
         ":package-srcs",
         "//cmd/kubeadm/app/util/apiclient:all-srcs",
         "//cmd/kubeadm/app/util/config:all-srcs",
+        "//cmd/kubeadm/app/util/dryrun:all-srcs",
         "//cmd/kubeadm/app/util/kubeconfig:all-srcs",
         "//cmd/kubeadm/app/util/pubkeypin:all-srcs",
         "//cmd/kubeadm/app/util/staticpod:all-srcs",

cmd/kubeadm/app/util/apiclient/BUILD

@@ -18,13 +18,13 @@ go_library(
     deps = [
         "//cmd/kubeadm/app/constants:go_default_library",
         "//pkg/registry/core/service/ipallocator:go_default_library",
-        "//vendor/github.com/ghodss/yaml:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
         "//vendor/k8s.io/api/rbac/v1beta1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",