Skip to content

Commit

Permalink
WIP minikube load support
Browse files Browse the repository at this point in the history
  • Loading branch information
@aaron-prindle
aaron-prindle committed Sep 13, 2021
1 parent e6c315b commit d78accd
Show file tree
Hide file tree
Showing 12 changed files with 271 additions and 76 deletions.
245 changes: 198 additions & 47 deletions pkg/skaffold/cluster/minikube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,22 @@ import (
"sync"

"github.com/blang/semver"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/util/homedir"

"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
kctx "github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes/context"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/output/log"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
"github.com/GoogleContainerTools/skaffold/pkg/skaffold/version"
)

var (
GetClient = getClient
GetClient = getClient
// --user flag introduced in 1.18.0
minikubeVersionWithUserFlag = semver.MustParse("1.18.0")
// `minikube profile list --light` introduced in 1.18.0
minikubeVersionWithProfileLightFlag = semver.MustParse("1.18.0")
)

// To override during tests
Expand All @@ -61,40 +65,83 @@ type Client interface {
IsMinikube(ctx context.Context, kubeContext string) bool
// MinikubeExec returns the Cmd struct to execute minikube with given arguments
MinikubeExec(ctx context.Context, arg ...string) (*exec.Cmd, error)
// UseDockerEnv() returns true if it is recommended to use Minikube's docker-env
UseDockerEnv(ctx context.Context) bool
}

type clientImpl struct{}
type clientImpl struct {
kubeContext string // empty if not resolved
isMinikube bool
profile *profile // nil if not yet fetched
}

func getClient() Client {
return clientImpl{}
return &clientImpl{}
}

func (clientImpl) IsMinikube(ctx context.Context, kubeContext string) bool {
// IsMinikube returns true if `kubeContext` seems to be a minikube cluster.
// This function is called on several hotpaths so it should be quick.
func (c *clientImpl) IsMinikube(ctx context.Context, kubeContext string) bool {
if c.kubeContext == kubeContext {
return c.isMinikube
}

c.kubeContext = kubeContext
c.isMinikube = false // default to not being minikube
c.profile = nil

if _, _, err := FindMinikubeBinary(ctx); err != nil {
log.Entry(context.TODO()).Tracef("Minikube cluster not detected: %v", err)
return false
}
// short circuit if context is 'minikube'
if kubeContext == constants.DefaultMinikubeContext {
return true
}

cluster, err := getClusterInfo(kubeContext)
// Although it's extremely unlikely that any other cluster would
// have the name `minikube`, our checks here are sufficiently quick
// that we can do a slightly more thorough check.

kubeConfig, err := kctx.CurrentConfig()
if err != nil {
log.Entry(context.TODO()).Tracef("Minikube cluster not detected: %v", err)
return false
}
ktxt, found := kubeConfig.Contexts[kubeContext]
if !found {
log.Entry(context.TODO()).Tracef("failed to get cluster info: %v", err)
return false
}
cluster, found := kubeConfig.Clusters[ktxt.Cluster]
if !found {
log.Entry(context.TODO()).Debugf("context %q could not be resolved to a cluster", kubeContext)
return false
}

if ext, found := ktxt.Extensions["context_info"]; found && isMinikubeExtension(ext) {
log.Entry(context.TODO()).Tracef("Minikube cluster detected: context %q has minikube `context_info`", kubeContext)
c.isMinikube = true
return true
}
if ext, found := ktxt.Extensions["cluster_info"]; found && isMinikubeExtension(ext) {
log.Entry(context.TODO()).Tracef("Minikube cluster detected: context %q cluster %q has minikube `cluster_info`", kubeContext, ktxt.Cluster)
c.isMinikube = true
return true
}
if matchClusterCertPath(cluster.CertificateAuthority) {
log.Entry(context.TODO()).Debugf("Minikube cluster detected: cluster certificate for context %q found inside the minikube directory", kubeContext)
log.Entry(context.TODO()).Tracef("Minikube cluster detected: context %q cluster %q has minikube certificate", kubeContext, ktxt.Cluster)
c.isMinikube = true
return true
}

if ok, err := matchServerURL(ctx, cluster.Server); err != nil {
log.Entry(context.TODO()).Tracef("failed to match server url: %v", err)
} else if ok {
log.Entry(context.TODO()).Debugf("Minikube cluster detected: server url for context %q matches minikube node ip", kubeContext)
if err := c.resolveProfile(ctx); err == nil {
log.Entry(context.TODO()).Tracef("Minikube cluster detected: context %q cluster %q resolved minikube profile", kubeContext, ktxt.Cluster)
c.isMinikube = true
return true
}
// if ok, err := matchServerURL(ctx, cluster.Server); err != nil {
// log.Entry(context.TODO()).Tracef("failed to match server url: %v", err)
// } else if ok {
// log.Entry(context.TODO()).Debugf("Minikube cluster detected: server url for context %q matches minikube node ip", kubeContext)
// return true
// }

log.Entry(context.TODO()).Tracef("Minikube cluster not detected for context %q", kubeContext)
return false
}
Expand All @@ -103,6 +150,90 @@ func (clientImpl) MinikubeExec(ctx context.Context, arg ...string) (*exec.Cmd, e
return minikubeExec(ctx, arg...)
}

func (c *clientImpl) UseDockerEnv(ctx context.Context) bool {
logrus.Tracef("Checking if build should use `minikube docker-env`")
if err := c.resolveProfile(ctx); err != nil {
logrus.Tracef("failed to match minikube profile: %v", err)
return false
}
// Cannot build to the docker daemon with multi-node setups
return c.profile.Config.KubernetesConfig.ContainerRuntime == "docker" && len(c.profile.Config.Nodes) == 1
}

// resolveProfile finds the matching minikube profile, a profile whose name or serverURL matches
// the selected context or cluster.
func (c *clientImpl) resolveProfile(ctx context.Context) error {
// early exit if already resolved
if c.profile != nil {
return nil
}

// TODO: Revisit once https://github.com/kubernetes/minikube/issues/6642 is fixed
kubeConfig, err := kctx.CurrentConfig()
if err != nil {
return err
}
ktxt, found := kubeConfig.Contexts[c.kubeContext]
if !found {
return fmt.Errorf("no context named %q", c.kubeContext)
}
cluster, found := kubeConfig.Clusters[ktxt.Cluster]
if !found {
return fmt.Errorf("no cluster %q found for context %q", ktxt.Cluster, c.kubeContext)
}

profiles, err := listProfilesLight(ctx)
if err != nil {
return fmt.Errorf("minikube profile list: %w", err)
}

// when minikube driver is a VM then the node IP should also match the k8s api server url
serverURL, err := url.Parse(cluster.Server)
if err != nil {
logrus.Tracef("invalid server url: %v", err)
return err
}
for _, v := range profiles.Valid {
for _, n := range v.Config.Nodes {
if serverURL.Host == fmt.Sprintf("%s:%d", n.IP, n.Port) {
c.profile = &v
return nil
}
}
}

// otherwise check for matching context or cluster name
for _, v := range profiles.Valid {
if v.Config.Name == ktxt.Cluster || v.Config.Name == c.kubeContext {
c.profile = &v
return nil
}
}

return fmt.Errorf("no valid minikube profile found for %q", c.kubeContext)
}

// listProfilesLight tries to get the minikube profiles as fast as possible.
func listProfilesLight(ctx context.Context) (*profileList, error) {
args := []string{"profile", "list", "-o", "json"}
if mk.version.GE(minikubeVersionWithProfileLightFlag) {
args = append(args, "--light")
}
cmd, err := minikubeExec(ctx, args...)
if err != nil {
return nil, fmt.Errorf("minikube profile list: %w", err)
}
out, err := util.RunCmdOut(ctx, cmd)
if err != nil {
return nil, fmt.Errorf("minikube profile list: %w", err)
}
var profiles profileList
if err = json.Unmarshal(out, &profiles); err != nil {
return nil, fmt.Errorf("unmarshal minikube profile list: %w", err)
}
return &profiles, nil
}

func minikubeExec(ctx context.Context, arg ...string) (*exec.Cmd, error) {
b, v, err := FindMinikubeBinary(ctx)
if err != nil && !errors.As(err, &versionErr{}) {
Expand Down Expand Up @@ -164,40 +295,54 @@ func (v versionErr) Error() string {
return v.err.Error()
}

// matchClusterCertPath checks if the cluster certificate for this context is from inside the minikube directory
func matchClusterCertPath(certPath string) bool {
return certPath != "" && util.IsSubPath(minikubePath(), certPath)
}

// matchServerURL checks if the k8s server url is same as any of the minikube nodes IPs
func matchServerURL(ctx context.Context, server string) (bool, error) {
cmd, _ := minikubeExec(ctx, "profile", "list", "-o", "json")
out, err := util.RunCmdOut(ctx, cmd)
if err != nil {
return false, fmt.Errorf("getting minikube profiles: %w", err)
}

var data profileList
if err = json.Unmarshal(out, &data); err != nil {
return false, fmt.Errorf("failed to unmarshal minikube profile list: %w", err)
// isMinikubeExtension checks if the provided extension is from minikube.
// This extension was introduced with minikube 1.17.
func isMinikubeExtension(extension runtime.Object) bool {
if extension == nil {
return false
}

serverURL, err := url.Parse(server)
m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(extension)
if err != nil {
log.Entry(context.TODO()).Tracef("invalid server url: %v", err)
logrus.Debugf("Unable to decode extension [%T]: %v", extension, err)
return false
}
return m["provider"] == "minikube.sigs.k8s.io"
}

for _, v := range data.Valid {
for _, n := range v.Config.Nodes {
if err == nil && serverURL.Host == fmt.Sprintf("%s:%d", n.IP, n.Port) {
// TODO: Revisit once https://github.com/kubernetes/minikube/issues/6642 is fixed
return true, nil
}
}
}
return false, nil
// matchClusterCertPath checks if the cluster certificate for this context is from inside the minikube directory
func matchClusterCertPath(certPath string) bool {
return certPath != "" && util.IsSubPath(minikubePath(), certPath)
}

// // matchServerURL checks if the k8s server url is same as any of the minikube nodes IPs
// func matchServerURL(ctx context.Context, server string) (bool, error) {
// cmd, _ := minikubeExec(ctx, "profile", "list", "-o", "json")
// out, err := util.RunCmdOut(ctx, cmd)
// if err != nil {
// return false, fmt.Errorf("getting minikube profiles: %w", err)
// }

// var data profileList
// if err = json.Unmarshal(out, &data); err != nil {
// return false, fmt.Errorf("failed to unmarshal minikube profile list: %w", err)
// }

// serverURL, err := url.Parse(server)
// if err != nil {
// log.Entry(context.TODO()).Tracef("invalid server url: %v", err)
// }

// for _, v := range data.Valid {
// for _, n := range v.Config.Nodes {
// if err == nil && serverURL.Host == fmt.Sprintf("%s:%d", n.IP, n.Port) {
// // TODO: Revisit once https://github.com/kubernetes/minikube/issues/6642 is fixed
// return true, nil
// }
// }
// }
// return false, nil
// }

// minikubePath returns the path to the user's minikube dir
func minikubePath() string {
minikubeHomeEnv := os.Getenv("MINIKUBE_HOME")
Expand All @@ -220,12 +365,18 @@ type profile struct {
}

type config struct {
Name string
Driver string
Nodes []node
Name string
// virtualbox, parallels, vmwarefusion, hyperkit, vmware, docker, ssh
Driver string
Nodes []node
KubernetesConfig kubernetesConfig
}

type node struct {
IP string
Port int32
}

type kubernetesConfig struct {
ContainerRuntime string
}
32 changes: 25 additions & 7 deletions pkg/skaffold/config/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,13 @@ func GetCluster(ctx context.Context, configFile string, minikubeProfile string,
}

kubeContext := cfg.Kubecontext
// TODO(aaron-prindle) refactor using this?
// isKindCluster, isK3dCluster := IsKindCluster(kubeContext), IsK3dCluster(kubeContext)
isKindCluster, isK3dCluster := IsKindCluster(kubeContext), IsK3dCluster(kubeContext)

var local bool
local := false
loadImages := false

switch {
case minikubeProfile != "":
local = true
Expand All @@ -217,14 +221,23 @@ func GetCluster(ctx context.Context, configFile string, minikubeProfile string,
log.Entry(context.TODO()).Infof("Using local-cluster=%t from config", *cfg.LocalCluster)
local = *cfg.LocalCluster

case kubeContext == constants.DefaultMinikubeContext ||
kubeContext == constants.DefaultDockerForDesktopContext ||
kubeContext == constants.DefaultDockerDesktopContext ||
isKindCluster || isK3dCluster:
case IsKindCluster(kubeContext):
local = true
loadImages = cfg.KindDisableLoad == nil || !*cfg.KindDisableLoad

case IsK3dCluster(kubeContext):
local = true
loadImages = cfg.K3dDisableLoad == nil || !*cfg.K3dDisableLoad

case kubeContext == constants.DefaultDockerForDesktopContext ||
kubeContext == constants.DefaultDockerDesktopContext:
local = true

case detectMinikube:
local = cluster.GetClient().IsMinikube(ctx, kubeContext)
minikubeClient := cluster.GetClient()
// minikubeClient := cluster.GetClient(kubeContext, minikubeProfile)
local = minikubeClient.IsMinikube(ctx, kubeContext)
loadImages = local && !minikubeClient.UseDockerEnv(ctx)

default:
local = false
Expand All @@ -234,7 +247,12 @@ func GetCluster(ctx context.Context, configFile string, minikubeProfile string,
k3dDisableLoad := cfg.K3dDisableLoad != nil && *cfg.K3dDisableLoad

// load images for local kind/k3d cluster unless explicitly disabled
loadImages := local && ((isKindCluster && !kindDisableLoad) || (isK3dCluster && !k3dDisableLoad))
// loadImages := local && ((isKindCluster && !kindDisableLoad) || (isK3dCluster && !k3dDisableLoad))

// TODO(aaron-prindle) refactor using this?
// loadImages := local && ((isKindCluster && !kindDisableLoad) || (isK3dCluster && !k3dDisableLoad)) ||
// (minikubeClient != nil && minikubeLoadImages)
loadImages = loadImages && local

// push images for remote cluster or local kind/k3d cluster with image loading disabled
pushImages := !local || (isKindCluster && kindDisableLoad) || (isK3dCluster && k3dDisableLoad)
Expand Down
4 changes: 2 additions & 2 deletions pkg/skaffold/deploy/component/kubernetes/component.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,9 @@ func newDebugger(mode config.RunMode, podSelector kubernetes.PodSelector, namesp
return debugging.NewContainerManager(podSelector, namespaces, kubeContext)
}

func newImageLoader(cfg k8sloader.Config, cli *kubectl.CLI) loader.ImageLoader {
func newImageLoader(cfg k8sloader.Config, cli *kubectl.CLI, minikubeProfile string) loader.ImageLoader {
if cfg.LoadImages() {
return k8sloader.NewImageLoader(cfg.GetKubeContext(), cli)
return k8sloader.NewImageLoader(cfg.GetKubeContext(), cli, minikubeProfile)
}
return &loader.NoopImageLoader{}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/skaffold/deploy/helm/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ func NewDeployer(ctx context.Context, cfg Config, labeller *label.DefaultLabelle
namespaces: &namespaces,
accessor: component.NewAccessor(cfg, cfg.GetKubeContext(), kubectl, podSelector, labeller, &namespaces),
debugger: component.NewDebugger(cfg.Mode(), podSelector, &namespaces, cfg.GetKubeContext()),
imageLoader: component.NewImageLoader(cfg, kubectl),
imageLoader: component.NewImageLoader(cfg, kubectl, cfg.MinikubeProfile()),
logger: logger,
statusMonitor: component.NewMonitor(cfg, cfg.GetKubeContext(), labeller, &namespaces),
syncer: component.NewSyncer(kubectl, &namespaces, logger.GetFormatter()),
Expand Down
Loading

0 comments on commit d78accd

Please sign in to comment.