start.go

/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/blang/semver/v4"
	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/host"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"golang.org/x/sync/errgroup"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/klog/v2"
	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
	"k8s.io/minikube/pkg/addons"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/kapi"
	"k8s.io/minikube/pkg/minikube/bootstrapper"
	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/cni"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/cruntime"
	"k8s.io/minikube/pkg/minikube/detect"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/kubeconfig"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/logs"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/mustload"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/out/register"
	"k8s.io/minikube/pkg/minikube/proxy"
	"k8s.io/minikube/pkg/minikube/reason"
	"k8s.io/minikube/pkg/minikube/registry"
	"k8s.io/minikube/pkg/minikube/style"
	"k8s.io/minikube/pkg/minikube/vmpath"
	"k8s.io/minikube/pkg/network"
	"k8s.io/minikube/pkg/util"
	"k8s.io/minikube/pkg/util/retry"
	kconst "k8s.io/minikube/third_party/kubeadm/app/constants"
)

const waitTimeout = "wait-timeout"

var (
	kicGroup   errgroup.Group
	cacheGroup errgroup.Group
)

// Starter is a struct with all the necessary information to start a node
type Starter struct {
	Runner         command.Runner
	PreExists      bool
	StopK8s        bool
	MachineAPI     libmachine.API
	Host           *host.Host
	Cfg            *config.ClusterConfig
	Node           *config.Node
	ExistingAddons map[string]bool
}

// Start spins up a guest and starts the Kubernetes node.
func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
	var wg sync.WaitGroup
	stopk8s, err := handleNoKubernetes(starter)
	if err != nil {
		return nil, err
	}
	if stopk8s {
		nv := semver.Version{Major: 0, Minor: 0, Patch: 0}
		cr := configureRuntimes(starter.Runner, *starter.Cfg, nv)

		showNoK8sVersionInfo(cr)

		configureMounts(&wg, *starter.Cfg)
		return nil, config.Write(viper.GetString(config.ProfileName), starter.Cfg)
	}

	// wait for preloaded tarball to finish downloading before configuring runtimes
	waitCacheRequiredImages(&cacheGroup)

	sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to parse Kubernetes version")
	}

	// configure the runtime (docker, containerd, crio)
	cr := configureRuntimes(starter.Runner, *starter.Cfg, sv)

	// check if installed runtime is compatible with current minikube code
	if err = cruntime.CheckCompatibility(cr); err != nil {
		return nil, err
	}

	showVersionInfo(starter.Node.KubernetesVersion, cr)

	// Add "host.minikube.internal" DNS alias (intentionally non-fatal)
	hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name)
	if err != nil {
		klog.Errorf("Unable to get host IP: %v", err)
	} else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil {
		klog.Errorf("Unable to add host alias: %v", err)
	}

	var kcs *kubeconfig.Settings
	var bs bootstrapper.Bootstrapper
	if apiServer {
		kcs, bs, err = handleAPIServer(starter, cr, hostIP)
		if err != nil {
			return nil, err
		}
	} else {
		bs, err = cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, starter.Runner)
		if err != nil {
			return nil, errors.Wrap(err, "Failed to get bootstrapper")
		}

		if err = bs.SetupCerts(*starter.Cfg, *starter.Node); err != nil {
			return nil, errors.Wrap(err, "setting up certs")
		}

		if err := bs.UpdateNode(*starter.Cfg, *starter.Node, cr); err != nil {
			return nil, errors.Wrap(err, "update node")
		}
	}

	go configureMounts(&wg, *starter.Cfg)

	wg.Add(1)
	go func() {
		defer wg.Done()
		profile, err := config.LoadProfile(starter.Cfg.Name)
		if err != nil {
			out.FailureT("Unable to load profile: {{.error}}", out.V{"error": err})
		}
		if err := CacheAndLoadImagesInConfig([]*config.Profile{profile}); err != nil {
			out.FailureT("Unable to push cached images: {{.error}}", out.V{"error": err})
		}
	}()

	// enable addons, both old and new!
	addonList := viper.GetStringSlice(config.AddonListFlag)
	if starter.ExistingAddons != nil {
		if viper.GetBool("force") {
			addons.Force = true
		}
		wg.Add(1)
		go addons.Start(&wg, starter.Cfg, starter.ExistingAddons, addonList)
	}

	// discourage use of the virtualbox driver
	if starter.Cfg.Driver == driver.VirtualBox && viper.GetBool(config.WantVirtualBoxDriverWarning) {
		warnVirtualBox()
	}

	if apiServer {
		// special ops for none , like change minikube directory.
		// multinode super doesn't work on the none driver
		if starter.Cfg.Driver == driver.None && len(starter.Cfg.Nodes) == 1 {
			prepareNone()
		}
	} else {
		// Make sure to use the command runner for the control plane to generate the join token
		cpBs, cpr, err := cluster.ControlPlaneBootstrapper(starter.MachineAPI, starter.Cfg, viper.GetString(cmdcfg.Bootstrapper))
		if err != nil {
			return nil, errors.Wrap(err, "getting control plane bootstrapper")
		}

		if err := joinCluster(starter, cpBs, bs); err != nil {
			return nil, errors.Wrap(err, "joining cp")
		}

		cnm, err := cni.New(starter.Cfg)
		if err != nil {
			return nil, errors.Wrap(err, "cni")
		}

		if err := cnm.Apply(cpr); err != nil {
			return nil, errors.Wrap(err, "cni apply")
		}
	}

	if !starter.Cfg.DisableOptimizations {
		// Scale down CoreDNS from default 2 to 1 replica.
		if err := kapi.ScaleDeployment(starter.Cfg.Name, meta.NamespaceSystem, kconst.CoreDNSDeploymentName, 1); err != nil {
			klog.Errorf("Unable to scale down deployment %q in namespace %q to 1 replica: %v", kconst.CoreDNSDeploymentName, meta.NamespaceSystem, err)
		}
	}

	klog.Infof("Will wait %s for node %+v", viper.GetDuration(waitTimeout), starter.Node)
	if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil {
		return nil, errors.Wrapf(err, "wait %s for node", viper.GetDuration(waitTimeout))
	}

	klog.Infof("waiting for startup goroutines ...")
	wg.Wait()

	// Write enabled addons to the config before completion
	return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg)
}

// handleNoKubernetes handles starting minikube without Kubernetes.
func handleNoKubernetes(starter Starter) (bool, error) {
	// Do not bootstrap cluster if --no-kubernetes.
	if starter.Node.KubernetesVersion == constants.NoKubernetesVersion {
		// Stop existing Kubernetes node if applicable.
		if starter.StopK8s {
			cr, err := cruntime.New(cruntime.Config{Type: starter.Cfg.KubernetesConfig.ContainerRuntime, Runner: starter.Runner, Socket: starter.Cfg.KubernetesConfig.CRISocket})
			if err != nil {
				return false, err
			}
			kubeadm.StopKubernetes(starter.Runner, cr)
		}
		return true, config.Write(viper.GetString(config.ProfileName), starter.Cfg)
	}
	return false, nil
}

// handleAPIServer handles starting the API server.
func handleAPIServer(starter Starter, cr cruntime.Manager, hostIP net.IP) (*kubeconfig.Settings, bootstrapper.Bootstrapper, error) {
	var err error

	// Must be written before bootstrap, otherwise health checks may flake due to stale IP.
	kcs := setupKubeconfig(starter.Host, starter.Cfg, starter.Node, starter.Cfg.Name)
	if err != nil {
		return nil, nil, errors.Wrap(err, "Failed to setup kubeconfig")
	}

	// Setup kubeadm (must come after setupKubeconfig).
	bs := setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner)
	err = bs.StartCluster(*starter.Cfg)
	if err != nil {
		ExitIfFatal(err, false)
		out.LogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, *starter.Cfg, starter.Runner))
		return nil, bs, err
	}

	// Write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper.
	if err := kubeconfig.Update(kcs); err != nil {
		return nil, bs, errors.Wrap(err, "Failed kubeconfig update")
	}

	// Not running this in a Go func can result in DNS answering taking up to 38 seconds, with the Go func it takes 6-10 seconds.
	go func() {
		// Inject {"host.minikube.internal": hostIP} record into CoreDNS.
		if err := addCoreDNSEntry(starter.Runner, "host.minikube.internal", hostIP.String(), *starter.Cfg); err != nil {
			klog.Warningf("Unable to inject {%q: %s} record into CoreDNS: %v", "host.minikube.internal", hostIP.String(), err)
			out.Err("Failed to inject host.minikube.internal into CoreDNS, this will limit the pods access to the host IP")
		}
	}()
	return kcs, bs, nil
}

// joinCluster adds new or prepares and then adds existing node to the cluster.
func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrapper.Bootstrapper) error {
	start := time.Now()
	klog.Infof("JoinCluster: %+v", starter.Cfg)
	defer func() {
		klog.Infof("JoinCluster complete in %s", time.Since(start))
	}()

	joinCmd, err := cpBs.GenerateToken(*starter.Cfg)
	if err != nil {
		return fmt.Errorf("error generating join token: %w", err)
	}

	// avoid "error execution phase kubelet-start: a Node with name "<name>" and status "Ready" already exists in the cluster.
	// You must delete the existing Node or change the name of this new joining Node"
	if starter.PreExists {
		klog.Infof("removing existing worker node %q before attempting to rejoin cluster: %+v", starter.Node.Name, starter.Node)
		if _, err := drainNode(*starter.Cfg, starter.Node.Name); err != nil {
			klog.Errorf("error removing existing worker node before rejoining cluster, will continue anyway: %v", err)
		}
		klog.Infof("successfully removed existing worker node %q from cluster: %+v", starter.Node.Name, starter.Node)
	}

	join := func() error {
		klog.Infof("trying to join worker node %q to cluster: %+v", starter.Node.Name, starter.Node)
		if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil {
			klog.Errorf("worker node failed to join cluster, will retry: %v", err)

			// reset worker node to revert any changes made by previous kubeadm init/join
			klog.Infof("resetting worker node %q before attempting to rejoin cluster...", starter.Node.Name)
			if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil {
				klog.Infof("kubeadm reset failed, continuing anyway: %v", err)
			} else {
				klog.Infof("successfully reset worker node %q", starter.Node.Name)
			}

			return err
		}
		return nil
	}
	if err := retry.Expo(join, 10*time.Second, 3*time.Minute); err != nil {
		return fmt.Errorf("error joining worker node to cluster: %w", err)
	}

	return nil
}

// Provision provisions the machine/container for the node
func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool, delOnFail bool) (command.Runner, bool, libmachine.API, *host.Host, error) {
	register.Reg.SetStep(register.StartingNode)
	name := config.MachineName(*cc, *n)

	// Be explicit with each case for the sake of translations
	if cc.KubernetesConfig.KubernetesVersion == constants.NoKubernetesVersion {
		out.Step(style.ThumbsUp, "Starting minikube without Kubernetes in cluster {{.cluster}}", out.V{"cluster": cc.Name})
	} else {
		if apiServer {
			out.Step(style.ThumbsUp, "Starting control plane node {{.name}} in cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
		} else {
			out.Step(style.ThumbsUp, "Starting worker node {{.name}} in cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
		}

	}

	if driver.IsKIC(cc.Driver) {
		beginDownloadKicBaseImage(&kicGroup, cc, viper.GetBool("download-only"))
	}

	if !driver.BareMetal(cc.Driver) {
		beginCacheKubernetesImages(&cacheGroup, cc.KubernetesConfig.ImageRepository, n.KubernetesVersion, cc.KubernetesConfig.ContainerRuntime, cc.Driver)
	}

	// Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.
	// Hence, SaveProfile must be called before startHost, and again afterwards when we know the IP.
	if err := config.SaveProfile(viper.GetString(config.ProfileName), cc); err != nil {
		return nil, false, nil, nil, errors.Wrap(err, "Failed to save config")
	}

	handleDownloadOnly(&cacheGroup, &kicGroup, n.KubernetesVersion, cc.KubernetesConfig.ContainerRuntime, cc.Driver)
	waitDownloadKicBaseImage(&kicGroup)

	return startMachine(cc, n, delOnFail)
}

// ConfigureRuntimes does what needs to happen to get a runtime going.
func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, kv semver.Version) cruntime.Manager {
	co := cruntime.Config{
		Type:              cc.KubernetesConfig.ContainerRuntime,
		Socket:            cc.KubernetesConfig.CRISocket,
		Runner:            runner,
		ImageRepository:   cc.KubernetesConfig.ImageRepository,
		KubernetesVersion: kv,
		InsecureRegistry:  cc.InsecureRegistry,
	}
	cr, err := cruntime.New(co)
	if err != nil {
		exit.Error(reason.InternalRuntime, "Failed runtime", err)
	}

	// 87-podman.conflist cni conf potentially conflicts with others and is created by podman on its first invocation,
	// so we "provoke" it here to ensure it's generated and that we can disable it
	// note: using 'help' or '--help' would be cheaper, but does not trigger that; 'version' seems to be next best option
	if co.Type == constants.CRIO {
		_, _ = runner.RunCmd(exec.Command("sudo", "sh", "-c", `podman version >/dev/null`))
	}
	// ensure loopback is properly configured
	// make sure container runtime is restarted afterwards for these changes to take effect
	disableLoopback := co.Type == constants.CRIO
	if err := cni.ConfigureLoopbackCNI(runner, disableLoopback); err != nil {
		klog.Warningf("unable to name loopback interface in dockerConfigureNetworkPlugin: %v", err)
	}
	if kv.GTE(semver.MustParse("1.24.0-alpha.2")) {
		if err := cruntime.ConfigureNetworkPlugin(cr, runner, cc.KubernetesConfig.NetworkPlugin); err != nil {
			exit.Error(reason.RuntimeEnable, "Failed to configure network plugin", err)
		}
	}
	// ensure all default CNI(s) are properly configured on each and every node (re)start
	// make sure container runtime is restarted afterwards for these changes to take effect
	if err := cni.ConfigureDefaultBridgeCNIs(runner, cc.KubernetesConfig.NetworkPlugin); err != nil {
		klog.Errorf("unable to disable preinstalled bridge CNI(s): %v", err)
	}

	// Preload is overly invasive for bare metal, and caching is not meaningful.
	// KIC handles preload elsewhere.
	if driver.IsVM(cc.Driver) {
		if err := cr.Preload(cc); err != nil {
			switch err.(type) {
			case *cruntime.ErrISOFeature:
				out.ErrT(style.Tip, "Existing disk is missing new features ({{.error}}). To upgrade, run 'minikube delete'", out.V{"error": err})
			default:
				klog.Warningf("%s preload failed: %v, falling back to caching images", cr.Name(), err)
			}

			if err := machine.CacheImagesForBootstrapper(cc.KubernetesConfig.ImageRepository, cc.KubernetesConfig.KubernetesVersion, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
				exit.Error(reason.RuntimeCache, "Failed to cache images", err)
			}
		}
	}

	inUserNamespace := strings.Contains(cc.KubernetesConfig.FeatureGates, "KubeletInUserNamespace=true")
	// for docker container runtime: ensure containerd is properly configured by calling Enable(), as docker could be bound to containerd
	// it will also "soft" start containerd, but it will not disable others; docker will disable containerd if not used in the next step
	if co.Type == constants.Docker {
		containerd, err := cruntime.New(cruntime.Config{
			Type:              constants.Containerd,
			Socket:            "", // use default
			Runner:            co.Runner,
			ImageRepository:   co.ImageRepository,
			KubernetesVersion: co.KubernetesVersion,
			InsecureRegistry:  co.InsecureRegistry})
		if err == nil {
			err = containerd.Enable(false, cgroupDriver(cc), inUserNamespace) // do not disableOthers, as it's not primary cr
		}
		if err != nil {
			klog.Warningf("cannot ensure containerd is configured properly and reloaded for docker - cluster might be unstable: %v", err)
		}
	}

	disableOthers := !driver.BareMetal(cc.Driver)
	if err = cr.Enable(disableOthers, cgroupDriver(cc), inUserNamespace); err != nil {
		exit.Error(reason.RuntimeEnable, "Failed to enable container runtime", err)
	}

	// Wait for the CRI to be "live", before returning it
	if err = waitForCRISocket(runner, cr.SocketPath(), 60, 1); err != nil {
		exit.Error(reason.RuntimeEnable, "Failed to start container runtime", err)
	}

	// Wait for the CRI to actually work, before returning
	if err = waitForCRIVersion(runner, cr.SocketPath(), 60, 10); err != nil {
		exit.Error(reason.RuntimeEnable, "Failed to start container runtime", err)
	}

	return cr
}

// cgroupDriver returns cgroup driver that should be used to further configure container runtime, node(s) and cluster.
// It is based on:
// - (forced) user preference (set via flags or env), if present, or
// - default settings for vm or ssh driver, if user, or
// - host os config detection, if possible.
// Possible mappings are: "v1" (legacy) cgroups => "cgroupfs", "v2" (unified) cgroups => "systemd" and "" (unknown) cgroups => constants.DefaultCgroupDriver.
// Note: starting from k8s v1.22, "kubeadm clusters should be using the systemd driver":
// ref: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.22.md#no-really-you-must-read-this-before-you-upgrade
// ref: https://kubernetes.io/docs/setup/production-environment/container-runtimes/#cgroup-drivers
// ref: https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/configure-cgroup-driver/
func cgroupDriver(cc config.ClusterConfig) string {
	klog.Info("detecting cgroup driver to use...")

	// check flags for user preference
	if viper.GetBool("force-systemd") {
		klog.Infof("using %q cgroup driver as enforced via flags", constants.SystemdCgroupDriver)
		return constants.SystemdCgroupDriver
	}

	// check env for user preference
	env := os.Getenv(constants.MinikubeForceSystemdEnv)
	if force, err := strconv.ParseBool(env); env != "" && err == nil && force {
		klog.Infof("using %q cgroup driver as enforced via env", constants.SystemdCgroupDriver)
		return constants.SystemdCgroupDriver
	}

	// vm driver uses iso that boots with cgroupfs cgroup driver by default atm (keep in sync!)
	if driver.IsVM(cc.Driver) {
		return constants.CgroupfsCgroupDriver
	}

	// for "remote baremetal", we assume cgroupfs and user can "force-systemd" with flag to override
	// potential improvement: use systemd as default (in line with k8s) and allow user to override it with new flag (eg, "cgroup-driver", that would replace "force-systemd")
	if driver.IsSSH(cc.Driver) {
		return constants.CgroupfsCgroupDriver
	}

	// in all other cases - try to detect and use what's on user's machine
	return detect.CgroupDriver()
}

func pathExists(runner cruntime.CommandRunner, path string) (bool, error) {
	_, err := runner.RunCmd(exec.Command("stat", path))
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func waitForCRISocket(runner cruntime.CommandRunner, socket string, wait int, interval int) error {

	if socket == "" || socket == "/var/run/dockershim.sock" {
		return nil
	}

	klog.Infof("Will wait %ds for socket path %s", wait, socket)

	chkPath := func() error {
		e, err := pathExists(runner, socket)
		if err != nil {
			return err
		}
		if !e {
			return &retry.RetriableError{Err: err}
		}
		return nil
	}
	return retry.Expo(chkPath, time.Duration(interval)*time.Second, time.Duration(wait)*time.Second)
}

func waitForCRIVersion(runner cruntime.CommandRunner, socket string, wait int, interval int) error {

	if socket == "" || socket == "/var/run/dockershim.sock" {
		return nil
	}

	klog.Infof("Will wait %ds for crictl version", wait)

	cmd := exec.Command("which", "crictl")
	rr, err := runner.RunCmd(cmd)
	if err != nil {
		return err
	}
	crictl := strings.TrimSuffix(rr.Stdout.String(), "\n")

	chkInfo := func() error {
		args := []string{crictl, "version"}
		cmd := exec.Command("sudo", args...)
		rr, err := runner.RunCmd(cmd)
		if err != nil && !os.IsNotExist(err) {
			return &retry.RetriableError{Err: err}
		}
		klog.Info(rr.Stdout.String())
		return nil
	}
	return retry.Expo(chkInfo, time.Duration(interval)*time.Second, time.Duration(wait)*time.Second)
}

// setupKubeAdm adds any requested files into the VM before Kubernetes is started
func setupKubeAdm(mAPI libmachine.API, cfg config.ClusterConfig, n config.Node, r command.Runner) bootstrapper.Bootstrapper {
	bs, err := cluster.Bootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper), cfg, r)
	if err != nil {
		exit.Error(reason.InternalBootstrapper, "Failed to get bootstrapper", err)
	}
	for _, eo := range cfg.KubernetesConfig.ExtraOptions {
		out.Infof("{{.extra_option_component_name}}.{{.key}}={{.value}}", out.V{"extra_option_component_name": eo.Component, "key": eo.Key, "value": eo.Value})
	}
	// Loads cached images, generates config files, download binaries
	// update cluster and set up certs

	if err := bs.UpdateCluster(cfg); err != nil {
		if errors.Is(err, cruntime.ErrContainerRuntimeNotRunning) {
			exit.Error(reason.KubernetesInstallFailedRuntimeNotRunning, "Failed to update cluster", err)
		}
		exit.Error(reason.KubernetesInstallFailed, "Failed to update cluster", err)
	}

	if err := bs.SetupCerts(cfg, n); err != nil {
		exit.Error(reason.GuestCert, "Failed to setup certs", err)
	}

	return bs
}

func setupKubeconfig(h *host.Host, cc *config.ClusterConfig, n *config.Node, clusterName string) *kubeconfig.Settings {
	addr, err := apiServerURL(*h, *cc, *n)
	if err != nil {
		exit.Message(reason.DrvCPEndpoint, fmt.Sprintf("failed to get API Server URL: %v", err), out.V{"profileArg": fmt.Sprintf("--profile=%s", clusterName)})
	}

	if cc.KubernetesConfig.APIServerName != constants.APIServerName {
		addr = strings.ReplaceAll(addr, n.IP, cc.KubernetesConfig.APIServerName)
	}
	kcs := &kubeconfig.Settings{
		ClusterName:          clusterName,
		Namespace:            cc.KubernetesConfig.Namespace,
		ClusterServerAddress: addr,
		ClientCertificate:    localpath.ClientCert(cc.Name),
		ClientKey:            localpath.ClientKey(cc.Name),
		CertificateAuthority: localpath.CACert(),
		KeepContext:          cc.KeepContext,
		EmbedCerts:           cc.EmbedCerts,
	}

	kcs.SetPath(kubeconfig.PathFromEnv())
	return kcs
}

func apiServerURL(h host.Host, cc config.ClusterConfig, n config.Node) (string, error) {
	hostname, _, port, err := driver.ControlPlaneEndpoint(&cc, &n, h.DriverName)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("https://" + net.JoinHostPort(hostname, strconv.Itoa(port))), nil
}

// StartMachine starts a VM
func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host, err error) {
	m, err := machine.NewAPIClient()
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to get machine client")
	}
	host, preExists, err = startHostInternal(m, cfg, node, delOnFail)
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to start host")
	}
	runner, err = machine.CommandRunner(host)
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to get command runner")
	}

	ip, err := validateNetwork(host, runner, cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network")
	}

	if driver.IsQEMU(host.Driver.DriverName()) && network.IsUser(cfg.Network) {
		apiServerPort, err := getPort()
		if err != nil {
			return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port")
		}
		cfg.APIServerPort = apiServerPort
	}

	// Bypass proxy for minikube's vm host ip
	err = proxy.ExcludeIP(ip)
	if err != nil {
		out.FailureT("Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.", out.V{"ip": ip})
	}

	return runner, preExists, m, host, err
}

// getPort asks the kernel for a free open port that is ready to use
func getPort() (int, error) {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
		panic(err)
	}

	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return -1, errors.Errorf("Error accessing port %d", addr.Port)
	}
	defer l.Close()
	return l.Addr().(*net.TCPAddr).Port, nil
}

// startHostInternal starts a new minikube host using a VM or None
func startHostInternal(api libmachine.API, cc *config.ClusterConfig, n *config.Node, delOnFail bool) (*host.Host, bool, error) {
	host, exists, err := machine.StartHost(api, cc, n)
	if err == nil {
		return host, exists, nil
	}
	klog.Warningf("error starting host: %v", err)
	// NOTE: People get very cranky if you delete their preexisting VM. Only delete new ones.
	if !exists {
		err := machine.DeleteHost(api, config.MachineName(*cc, *n))
		if err != nil {
			klog.Warningf("delete host: %v", err)
		}
	}

	if err, ff := errors.Cause(err).(*oci.FailFastError); ff {
		klog.Infof("will skip retrying to create machine because error is not retriable: %v", err)
		return host, exists, err
	}

	out.ErrT(style.Embarrassed, "StartHost failed, but will try again: {{.error}}", out.V{"error": err})
	klog.Info("Will try again in 5 seconds ...")
	// Try again, but just once to avoid making the logs overly confusing
	time.Sleep(5 * time.Second)

	if delOnFail {
		klog.Info("Deleting existing host since delete-on-failure was set.")
		// Delete the failed existing host
		err := machine.DeleteHost(api, config.MachineName(*cc, *n))
		if err != nil {
			klog.Warningf("delete host: %v", err)
		}
	}

	host, exists, err = machine.StartHost(api, cc, n)
	if err == nil {
		return host, exists, nil
	}

	// Don't use host.Driver to avoid nil pointer deref
	drv := cc.Driver
	out.ErrT(style.Sad, `Failed to start {{.driver}} {{.driver_type}}. Running "{{.cmd}}" may fix it: {{.error}}`, out.V{"driver": drv, "driver_type": driver.MachineType(drv), "cmd": mustload.ExampleCmd(cc.Name, "delete"), "error": err})
	return host, exists, err
}

// validateNetwork tries to catch network problems as soon as possible
func validateNetwork(h *host.Host, r command.Runner, imageRepository string, kubernetesVersion string) (string, error) {
	ip, err := h.Driver.GetIP()
	if err != nil {
		return ip, err
	}

	optSeen := false
	warnedOnce := false
	for _, k := range proxy.EnvVars {
		if v := os.Getenv(k); v != "" {
			if !optSeen {
				out.Styled(style.Internet, "Found network options:")
				optSeen = true
			}
			out.Infof("{{.key}}={{.value}}", out.V{"key": k, "value": v})
			ipExcluded := proxy.IsIPExcluded(ip) // Skip warning if minikube ip is already in NO_PROXY
			k = strings.ToUpper(k)               // for http_proxy & https_proxy
			if (k == "HTTP_PROXY" || k == "HTTPS_PROXY") && !ipExcluded && !warnedOnce {
				out.WarningT("You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}).", out.V{"ip_address": ip})
				out.Styled(style.Documentation, "Please see {{.documentation_url}} for more details", out.V{"documentation_url": "https://minikube.sigs.k8s.io/docs/handbook/vpn_and_proxy/"})
				warnedOnce = true
			}
		}
	}

	if shouldTrySSH(h.Driver.DriverName(), ip) {
		if err := trySSH(h, ip); err != nil {
			return ip, err
		}
	}

	// Non-blocking
	go tryRegistry(r, h.Driver.DriverName(), imageRepository, kubernetesVersion, ip)
	return ip, nil
}

func shouldTrySSH(driverName, ip string) bool {
	if driver.BareMetal(driverName) || driver.IsKIC(driverName) {
		return false
	}
	// QEMU with user network
	if driver.IsQEMU(driverName) && ip == "127.0.0.1" {
		return false
	}
	return true
}

func trySSH(h *host.Host, ip string) error {
	if viper.GetBool("force") {
		return nil
	}

	sshAddr := net.JoinHostPort(ip, "22")

	dial := func() (err error) {
		d := net.Dialer{Timeout: 3 * time.Second}
		conn, err := d.Dial("tcp", sshAddr)
		if err != nil {
			klog.Warningf("dial failed (will retry): %v", err)
			return err
		}
		_ = conn.Close()
		return nil
	}

	err := retry.Expo(dial, time.Second, 13*time.Second)
	if err != nil {
		out.ErrT(style.Failure, `minikube is unable to connect to the VM: {{.error}}

	This is likely due to one of two reasons:

	- VPN or firewall interference
	- {{.hypervisor}} network configuration issue

	Suggested workarounds:

	- Disable your local VPN or firewall software
	- Configure your local VPN or firewall to allow access to {{.ip}}
	- Restart or reinstall {{.hypervisor}}
	- Use an alternative --vm-driver
	- Use --force to override this connectivity check
	`, out.V{"error": err, "hypervisor": h.Driver.DriverName(), "ip": ip})
	}

	return err
}

// tryRegistry tries to connect to the image repository
func tryRegistry(r command.Runner, driverName string, imageRepository string, kubernetesVersion string, ip string) {
	// 2 second timeout. For best results, call tryRegistry in a non-blocking manner.
	opts := []string{"-sS", "-m", "2"}

	proxy := os.Getenv("HTTPS_PROXY")
	if proxy != "" && !strings.HasPrefix(proxy, "localhost") && !strings.HasPrefix(proxy, "127.0") {
		opts = append([]string{"-x", proxy}, opts...)
	}

	if imageRepository == "" {
		v, _ := util.ParseKubernetesVersion(kubernetesVersion)
		imageRepository = images.DefaultKubernetesRepo(v)
	}

	opts = append(opts, fmt.Sprintf("https://%s/", imageRepository))
	if rr, err := r.RunCmd(exec.Command("curl", opts...)); err != nil {
		klog.Warningf("%s failed: %v", rr.Args, err)
		out.WarningT("This {{.type}} is having trouble accessing https://{{.repository}}", out.V{"repository": imageRepository, "type": driver.MachineType(driverName)})
		out.ErrT(style.Tip, "To pull new external images, you may need to configure a proxy: https://minikube.sigs.k8s.io/docs/reference/networking/proxy/")
		// using QEMU with the user network
		if driver.IsQEMU(driverName) && ip == "127.0.0.1" {
			out.WarningT("Due to DNS issues your cluster may have problems starting and you may not be able to pull images\nMore details available at: https://minikube.sigs.k8s.io/docs/drivers/qemu/#known-issues")
		}
	}
}

// prepareNone prepares the user and host for the joy of the "none" driver
func prepareNone() {
	register.Reg.SetStep(register.ConfiguringLHEnv)
	out.Step(style.StartingNone, "Configuring local host environment ...")
	if viper.GetBool(config.WantNoneDriverWarning) {
		out.ErrT(style.Empty, "")
		out.WarningT("The 'none' driver is designed for experts who need to integrate with an existing VM")
		out.ErrT(style.Tip, "Most users should use the newer 'docker' driver instead, which does not require root!")
		out.ErrT(style.Documentation, "For more information, see: https://minikube.sigs.k8s.io/docs/reference/drivers/none/")
		out.ErrT(style.Empty, "")
	}

	if os.Getenv("CHANGE_MINIKUBE_NONE_USER") == "" {
		home := os.Getenv("HOME")
		out.WarningT("kubectl and minikube configuration will be stored in {{.home_folder}}", out.V{"home_folder": home})
		out.WarningT("To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:")

		out.ErrT(style.Empty, "")
		out.ErrT(style.Command, "sudo mv {{.home_folder}}/.kube {{.home_folder}}/.minikube $HOME", out.V{"home_folder": home})
		out.ErrT(style.Command, "sudo chown -R $USER $HOME/.kube $HOME/.minikube")
		out.ErrT(style.Empty, "")

		out.ErrT(style.Tip, "This can also be done automatically by setting the env var CHANGE_MINIKUBE_NONE_USER=true")
	}

	if err := util.MaybeChownDirRecursiveToMinikubeUser(localpath.MiniPath()); err != nil {
		exit.Message(reason.HostHomeChown, "Failed to change permissions for {{.minikube_dir_path}}: {{.error}}", out.V{"minikube_dir_path": localpath.MiniPath(), "error": err})
	}
}

// addCoreDNSEntry adds host name and IP record to the DNS by updating CoreDNS's ConfigMap.
// ref: https://coredns.io/plugins/hosts/
// note: there can be only one 'hosts' block in CoreDNS's ConfigMap (avoid "plugin/hosts: this plugin can only be used once per Server Block" error)
func addCoreDNSEntry(runner command.Runner, name, ip string, cc config.ClusterConfig) error {
	kubectl := kapi.KubectlBinaryPath(cc.KubernetesConfig.KubernetesVersion)
	kubecfg := path.Join(vmpath.GuestPersistentDir, "kubeconfig")

	// get current coredns configmap via kubectl
	get := fmt.Sprintf("sudo %s --kubeconfig=%s -n kube-system get configmap coredns -o yaml", kubectl, kubecfg)
	out, err := runner.RunCmd(exec.Command("/bin/bash", "-c", get))
	if err != nil {
		klog.Errorf("failed to get current CoreDNS ConfigMap: %v", err)
		return err
	}
	cm := strings.TrimSpace(out.Stdout.String())

	// check if this specific host entry already exists in coredns configmap, so not to duplicate/override it
	host := regexp.MustCompile(fmt.Sprintf(`(?smU)^ *hosts {.*%s.*}`, name))
	if host.MatchString(cm) {
		klog.Infof("CoreDNS already contains %q host record, skipping...", name)
		return nil
	}

	// inject hosts block with host record into coredns configmap
	sed := fmt.Sprintf("sed -e '/^        forward . \\/etc\\/resolv.conf.*/i \\        hosts {\\n           %s %s\\n           fallthrough\\n        }'", ip, name)
	// check if hosts block already exists in coredns configmap
	hosts := regexp.MustCompile(`(?smU)^ *hosts {.*}`)
	if hosts.MatchString(cm) {
		// inject host record into existing coredns configmap hosts block instead
		klog.Info("CoreDNS already contains hosts block, will inject host record there...")
		sed = fmt.Sprintf("sed -e '/^        hosts {.*/a \\           %s %s'", ip, name)
	}

	// check if logging is already enabled (via log plugin) in coredns configmap, so not to duplicate it
	logs := regexp.MustCompile(`(?smU)^ *log *$`)
	if !logs.MatchString(cm) {
		// inject log plugin into coredns configmap
		sed = fmt.Sprintf("%s -e '/^        errors *$/i \\        log'", sed)
	}

	// replace coredns configmap via kubectl
	replace := fmt.Sprintf("sudo %s --kubeconfig=%s replace -f -", kubectl, kubecfg)
	if _, err := runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s | %s | %s", get, sed, replace))); err != nil {
		klog.Errorf("failed to inject {%q: %s} host record into CoreDNS", name, ip)
		return err
	}
	klog.Infof("{%q: %s} host record injected into CoreDNS's ConfigMap", name, ip)

	return nil
}

// prints a warning to the console against the use of the 'virtualbox' driver, if alternatives are available and healthy
func warnVirtualBox() {
	var altDriverList strings.Builder
	for _, choice := range driver.Choices(true) {
		if choice.Name != "virtualbox" && choice.Priority != registry.Discouraged && choice.State.Installed && choice.State.Healthy {
			altDriverList.WriteString(fmt.Sprintf("\n\t- %s", choice.Name))
		}
	}

	if altDriverList.Len() != 0 {
		out.Boxed(`You have selected "virtualbox" driver, but there are better options !
For better performance and support consider using a different driver: {{.drivers}}

To turn off this warning run:

	$ minikube config set WantVirtualBoxDriverWarning false


To learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/
To see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/

`, out.V{"drivers": altDriverList.String()})
	}
}