From e6e4d2a137f1df35315e4334e8f3b443bdc4dda8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 10 Apr 2021 20:00:38 +0200 Subject: [PATCH 1/4] Allow using podman as well as docker daemon --- pkg/minikube/image/image.go | 98 ++++++++++++++++++++++++---- pkg/minikube/machine/cache_images.go | 13 +++- pkg/minikube/node/cache.go | 27 ++++---- 3 files changed, 109 insertions(+), 29 deletions(-) diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index bc32f5bb052b..389a40970881 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -19,6 +19,7 @@ package image import ( "context" "fmt" + "io" "io/ioutil" "os" "os/exec" @@ -84,6 +85,17 @@ func DigestByDockerLib(imgClient *client.Client, imgName string) string { return img.ID } +// DigestByPodmanExec uses podman to return image digest +func DigestByPodmanExec(imgName string) string { + cmd := exec.Command("sudo", "-n", "podman", "image", "inspect", "--format", "{{.Id}}", imgName) + output, err := cmd.Output() + if err != nil { + klog.Infof("couldn't find image digest %s from local podman: %v ", imgName, err) + return "" + } + return strings.TrimSpace(string(output)) +} + // DigestByGoLib gets image digest uses go-containerregistry lib // which is 4s slower thabn local daemon per lookup https://github.com/google/go-containerregistry/issues/627 func DigestByGoLib(imgName string) string { @@ -125,30 +137,76 @@ func ExistsImageInCache(img string) bool { } // ExistsImageInDaemon if img exist in local docker daemon -func ExistsImageInDaemon(img string) bool { +func ExistsImageInDaemon(binary, img string) bool { // Check if image exists locally - klog.Infof("Checking for %s in local docker daemon", img) - cmd := exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}@{{.Digest}}") - if output, err := cmd.Output(); err == nil { - if strings.Contains(string(output), img) { - klog.Infof("Found %s in local docker daemon, skipping pull", img) - return true + switch binary { + case driver.Podman: + klog.Infof("Checking for %s in local podman", img) + cmd := exec.Command("sudo", "-n", "podman", "images", "--format", `{{$repository := .Repository}}{{$tag := .Tag}}{{range .RepoDigests}}{{$repository}}:{{$tag}}@{{.}}{{printf "\n"}}{{end}}`) + if output, err := cmd.Output(); err == nil { + if strings.Contains(string(output), img) { + klog.Infof("Found %s in local podman, skipping pull", img) + return true + } + } + case driver.Docker: + klog.Infof("Checking for %s in local docker daemon", img) + cmd := exec.Command("docker", "images", "--format", "{{.Repository}}:{{.Tag}}@{{.Digest}}") + if output, err := cmd.Output(); err == nil { + if strings.Contains(string(output), img) { + klog.Infof("Found %s in local docker daemon, skipping pull", img) + return true + } } } // Else, pull it return false } +// podmanWrite saves the image into podman as the given tag. +func podmanWrite(ref name.Reference, img v1.Image, opts ...tarball.WriteOption) (string, error) { + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(tarball.Write(ref, img, pw, opts...)) + }() + + // write the image in docker save format first, then load it + cmd := exec.Command("sudo", "podman", "image", "load") + cmd.Stdin = pr + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("error loading image: %v", err) + } + // pull the image from the registry, to get the digest too + // podman: "Docker references with both a tag and digest are currently not supported" + cmd = exec.Command("sudo", "podman", "image", "pull", strings.Split(ref.Name(), "@")[0]) + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("error pulling image: %v", err) + } + return string(output), nil +} + // LoadFromTarball checks if the image exists as a tarball and tries to load it to the local daemon -// TODO: Pass in if we are loading to docker or podman so this function can also be used for podman func LoadFromTarball(binary, img string) error { p := filepath.Join(constants.ImageCacheDir, img) p = localpath.SanitizeCacheDir(p) switch binary { case driver.Podman: - return fmt.Errorf("not yet implemented, see issue #8426") - default: + tag, err := name.NewTag(Tag(img)) + if err != nil { + return errors.Wrap(err, "new tag") + } + + i, err := tarball.ImageFromPath(p, &tag) + if err != nil { + return errors.Wrap(err, "tarball") + } + + _, err = podmanWrite(tag, i) + return err + case driver.Docker: tag, err := name.NewTag(Tag(img)) if err != nil { return errors.Wrap(err, "new tag") @@ -162,7 +220,7 @@ func LoadFromTarball(binary, img string) error { _, err = daemon.Write(tag, i) return err } - + return fmt.Errorf("unknown binary: %s", binary) } // Tag returns just the image with the tag @@ -243,11 +301,16 @@ func WriteImageToCache(img string) error { } // WriteImageToDaemon write img to the local docker daemon -func WriteImageToDaemon(img string) error { +func WriteImageToDaemon(binary, img string) error { // buffered channel c := make(chan v1.Update, 200) - klog.Infof("Writing %s to local daemon", img) + switch binary { + case driver.Podman: + klog.Infof("Writing %s to local podman", img) + case driver.Docker: + klog.Infof("Writing %s to local daemon", img) + } ref, err := name.ParseReference(img) if err != nil { return errors.Wrap(err, "parsing reference") @@ -281,7 +344,14 @@ func WriteImageToDaemon(img string) error { p.SetWidth(79) go func() { - _, err = daemon.Write(ref, i, tarball.WithProgress(c)) + switch binary { + case driver.Podman: + _, err = podmanWrite(ref, i, tarball.WithProgress(c)) + case driver.Docker: + _, err = daemon.Write(ref, i, tarball.WithProgress(c)) + default: + err = fmt.Errorf("unknown binary: %s", binary) + } errchan <- err }() var update v1.Update diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 9412dca1a614..a612a86bef31 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -138,8 +138,8 @@ func timedNeedsTransfer(imgClient *client.Client, imgName string, cr cruntime.Ma // needsTransfer returns an error if an image needs to be retransfered func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager) error { - imgDgst := "" // for instance sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed - if imgClient != nil { // if possible try to get img digest from Client lib which is 4s faster. + imgDgst := "" // for instance sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed + if cr.Name() == "docker" && imgClient != nil { // if possible try to get img digest from Client lib which is 4s faster. imgDgst = image.DigestByDockerLib(imgClient, imgName) if imgDgst != "" { if !cr.ImageExists(imgName, imgDgst) { @@ -148,6 +148,15 @@ func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager return nil } } + if cr.Name() == "podman" { + imgDgst = image.DigestByPodmanExec(imgName) + if imgDgst != "" { + if !cr.ImageExists(imgName, imgDgst) { + return fmt.Errorf("%q does not exist at hash %q in container runtime", imgName, imgDgst) + } + return nil + } + } // if not found with method above try go-container lib (which is 4s slower) imgDgst = image.DigestByGoLib(imgName) if imgDgst == "" { diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index 3ab2f7a96616..a3cd5a8a09e1 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -31,7 +31,6 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/download" - "k8s.io/minikube/pkg/minikube/driver" "k8s.io/minikube/pkg/minikube/exit" "k8s.io/minikube/pkg/minikube/image" "k8s.io/minikube/pkg/minikube/localpath" @@ -107,6 +106,10 @@ func doCacheBinaries(k8sVersion string) error { // beginDownloadKicBaseImage downloads the kic image func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, downloadOnly bool) { + if image.ExistsImageInDaemon(cc.Driver, cc.KicBaseImage) { + klog.Infof("%s exists in daemon, skipping pull", cc.KicBaseImage) + return + } klog.Infof("Beginning downloading kic base image for %s with %s", cc.Driver, cc.KubernetesConfig.ContainerRuntime) register.Reg.SetStep(register.PullingBaseImage) @@ -141,7 +144,6 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down if downloadOnly { return err } - if err := image.LoadFromTarball(cc.Driver, img); err == nil { klog.Infof("successfully loaded %s from cached tarball", img) // strip the digest from the img before saving it in the config @@ -150,17 +152,16 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down return nil } - if driver.IsDocker(cc.Driver) { - if image.ExistsImageInDaemon(img) { - klog.Infof("%s exists in daemon, skipping pull", img) - finalImg = img - return nil - } - - klog.Infof("Downloading %s to local daemon", img) - err = image.WriteImageToDaemon(img) - if err == nil { - klog.Infof("successfully downloaded %s", img) + klog.Infof("Downloading %s to local daemon", img) + err := image.WriteImageToDaemon(cc.Driver, img) + if err == nil { + klog.Infof("successfully downloaded %s", img) + finalImg = img + return nil + } + if downloadOnly { + if err := image.SaveToDir([]string{img}, constants.ImageCacheDir); err == nil { + klog.Infof("successfully saved %s as a tarball", img) finalImg = img return nil } From 54bc977a7429955bde7efb0e825f3143b64ca9b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 10 Apr 2021 20:21:57 +0200 Subject: [PATCH 2/4] Move podman.Write function into separate file Replaces daemon.Write from go-containerregistry --- pkg/minikube/image/image.go | 29 ++------------------ pkg/minikube/image/podman.go | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 27 deletions(-) create mode 100644 pkg/minikube/image/podman.go diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index 389a40970881..180ceb88fbc1 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -19,7 +19,6 @@ package image import ( "context" "fmt" - "io" "io/ioutil" "os" "os/exec" @@ -163,30 +162,6 @@ func ExistsImageInDaemon(binary, img string) bool { return false } -// podmanWrite saves the image into podman as the given tag. -func podmanWrite(ref name.Reference, img v1.Image, opts ...tarball.WriteOption) (string, error) { - pr, pw := io.Pipe() - go func() { - pw.CloseWithError(tarball.Write(ref, img, pw, opts...)) - }() - - // write the image in docker save format first, then load it - cmd := exec.Command("sudo", "podman", "image", "load") - cmd.Stdin = pr - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("error loading image: %v", err) - } - // pull the image from the registry, to get the digest too - // podman: "Docker references with both a tag and digest are currently not supported" - cmd = exec.Command("sudo", "podman", "image", "pull", strings.Split(ref.Name(), "@")[0]) - err = cmd.Run() - if err != nil { - return "", fmt.Errorf("error pulling image: %v", err) - } - return string(output), nil -} - // LoadFromTarball checks if the image exists as a tarball and tries to load it to the local daemon func LoadFromTarball(binary, img string) error { p := filepath.Join(constants.ImageCacheDir, img) @@ -204,7 +179,7 @@ func LoadFromTarball(binary, img string) error { return errors.Wrap(err, "tarball") } - _, err = podmanWrite(tag, i) + _, err = PodmanWrite(tag, i) return err case driver.Docker: tag, err := name.NewTag(Tag(img)) @@ -346,7 +321,7 @@ func WriteImageToDaemon(binary, img string) error { go func() { switch binary { case driver.Podman: - _, err = podmanWrite(ref, i, tarball.WithProgress(c)) + _, err = PodmanWrite(ref, i, tarball.WithProgress(c)) case driver.Docker: _, err = daemon.Write(ref, i, tarball.WithProgress(c)) default: diff --git a/pkg/minikube/image/podman.go b/pkg/minikube/image/podman.go new file mode 100644 index 000000000000..0e8e4c021615 --- /dev/null +++ b/pkg/minikube/image/podman.go @@ -0,0 +1,53 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package image + +import ( + "fmt" + "io" + "os/exec" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// PodmanWrite saves the image into podman as the given tag. +// same as github.com/google/go-containerregistry/pkg/v1/daemon +func PodmanWrite(ref name.Reference, img v1.Image, opts ...tarball.WriteOption) (string, error) { + pr, pw := io.Pipe() + go func() { + _ = pw.CloseWithError(tarball.Write(ref, img, pw, opts...)) + }() + + // write the image in docker save format first, then load it + cmd := exec.Command("sudo", "podman", "image", "load") + cmd.Stdin = pr + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("error loading image: %v", err) + } + // pull the image from the registry, to get the digest too + // podman: "Docker references with both a tag and digest are currently not supported" + cmd = exec.Command("sudo", "podman", "image", "pull", strings.Split(ref.Name(), "@")[0]) + err = cmd.Run() + if err != nil { + return "", fmt.Errorf("error pulling image: %v", err) + } + return string(output), nil +} From 90bff94fd62065f333219466dfb6c9e67b911433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 10 Apr 2021 21:39:28 +0200 Subject: [PATCH 3/4] Add podman.Image function to separate file Replaces daemon.Image from go-containerregistry --- pkg/minikube/image/image.go | 35 ++++++++++++++++++++-------- pkg/minikube/image/podman.go | 25 ++++++++++++++++++++ pkg/minikube/machine/cache_images.go | 2 +- pkg/minikube/node/cache.go | 3 +-- 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index 180ceb88fbc1..fef2790fad5c 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -50,6 +50,7 @@ const ( defaultDomain = "docker.io" ) +var daemonBinary string var defaultPlatform = v1.Platform{ Architecture: runtime.GOARCH, OS: "linux", @@ -97,13 +98,14 @@ func DigestByPodmanExec(imgName string) string { // DigestByGoLib gets image digest uses go-containerregistry lib // which is 4s slower thabn local daemon per lookup https://github.com/google/go-containerregistry/issues/627 -func DigestByGoLib(imgName string) string { +func DigestByGoLib(binary, imgName string) string { ref, err := name.ParseReference(imgName, name.WeakValidation) if err != nil { klog.Infof("error parsing image name %s ref %v ", imgName, err) return "" } + daemonBinary = binary img, _, err := retrieveImage(ref, imgName) if err != nil { klog.Infof("error retrieve Image %s ref %v ", imgName, err) @@ -379,7 +381,7 @@ func retrieveImage(ref name.Reference, imgName string) (v1.Image, string, error) klog.Infof("short name: %s", imgName) } } - img, err = retrieveDaemon(ref) + img, err = retrieveDaemon(daemonBinary, ref) if err == nil { return img, imgName, nil } @@ -397,15 +399,28 @@ func retrieveImage(ref name.Reference, imgName string) (v1.Image, string, error) return nil, "", err } -func retrieveDaemon(ref name.Reference) (v1.Image, error) { - img, err := daemon.Image(ref) - if err == nil { - klog.Infof("found %s locally: %+v", ref.Name(), img) - return img, nil +func retrieveDaemon(binary string, ref name.Reference) (v1.Image, error) { + switch binary { + case driver.Podman: + img, err := PodmanImage(ref) + if err == nil { + klog.Infof("found %s locally: %+v", ref.Name(), img) + return img, nil + } + // reference does not exist in the local podman + klog.Infof("podman lookup for %+v: %v", ref, err) + return img, err + case driver.Docker: + img, err := daemon.Image(ref) + if err == nil { + klog.Infof("found %s locally: %+v", ref.Name(), img) + return img, nil + } + // reference does not exist in the local daemon + klog.Infof("daemon lookup for %+v: %v", ref, err) + return img, err } - // reference does not exist in the local daemon - klog.Infof("daemon lookup for %+v: %v", ref, err) - return img, err + return nil, fmt.Errorf("unknown binary: %s", binary) } func retrieveRemote(ref name.Reference, p v1.Platform) (v1.Image, error) { diff --git a/pkg/minikube/image/podman.go b/pkg/minikube/image/podman.go index 0e8e4c021615..8abbc50fa1d5 100644 --- a/pkg/minikube/image/podman.go +++ b/pkg/minikube/image/podman.go @@ -27,6 +27,31 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" ) +// PodmanImage provides access to an image reference from podman. +// same as github.com/google/go-containerregistry/pkg/v1/daemon +func PodmanImage(ref name.Reference, options ...interface{}) (v1.Image, error) { + var img v1.Image + pr, pw := io.Pipe() + go func() { + opener := func() (io.ReadCloser, error) { + return pr, nil + } + var err error + tag := ref.(name.Digest).Tag() + img, err = tarball.Image(opener, &tag) + _ = pr.CloseWithError(err) + }() + + // write the image in docker save format first, then load it + cmd := exec.Command("sudo", "podman", "image", "save", strings.Split(ref.Name(), "@")[0]) + cmd.Stdout = pw + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("error loading image: %v", err) + } + return img, nil +} + // PodmanWrite saves the image into podman as the given tag. // same as github.com/google/go-containerregistry/pkg/v1/daemon func PodmanWrite(ref name.Reference, img v1.Image, opts ...tarball.WriteOption) (string, error) { diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index a612a86bef31..790074f641dd 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -158,7 +158,7 @@ func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager } } // if not found with method above try go-container lib (which is 4s slower) - imgDgst = image.DigestByGoLib(imgName) + imgDgst = image.DigestByGoLib(cr.Name(), imgName) if imgDgst == "" { return fmt.Errorf("got empty img digest %q for %s", imgDgst, imgName) } diff --git a/pkg/minikube/node/cache.go b/pkg/minikube/node/cache.go index a3cd5a8a09e1..1114959e385a 100644 --- a/pkg/minikube/node/cache.go +++ b/pkg/minikube/node/cache.go @@ -153,8 +153,7 @@ func beginDownloadKicBaseImage(g *errgroup.Group, cc *config.ClusterConfig, down } klog.Infof("Downloading %s to local daemon", img) - err := image.WriteImageToDaemon(cc.Driver, img) - if err == nil { + if err = image.WriteImageToDaemon(cc.Driver, img); err == nil { klog.Infof("successfully downloaded %s", img) finalImg = img return nil From 9a9e804e22cae23e0f644207a40f6c40b10ab1f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20F=20Bj=C3=B6rklund?= Date: Sat, 10 Apr 2021 22:00:24 +0200 Subject: [PATCH 4/4] Slight confusion between driver and runtime --- pkg/minikube/machine/cache_images.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index 790074f641dd..4466a01e0cb9 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -99,7 +99,7 @@ func LoadCachedImages(cc *config.ClusterConfig, runner command.Runner, images [] // because it takes much less than that time to just transfer the image. // This is needed because if running in offline mode, we can spend minutes here // waiting for i/o timeout. - err := timedNeedsTransfer(imgClient, image, cr, 10*time.Second) + err := timedNeedsTransfer(cc.Driver, imgClient, image, cr, 10*time.Second) if err == nil { return nil } @@ -114,7 +114,7 @@ func LoadCachedImages(cc *config.ClusterConfig, runner command.Runner, images [] return nil } -func timedNeedsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager, t time.Duration) error { +func timedNeedsTransfer(driver string, imgClient *client.Client, imgName string, cr cruntime.Manager, t time.Duration) error { timeout := make(chan bool, 1) go func() { time.Sleep(t) @@ -124,7 +124,7 @@ func timedNeedsTransfer(imgClient *client.Client, imgName string, cr cruntime.Ma transferFinished := make(chan bool, 1) var err error go func() { - err = needsTransfer(imgClient, imgName, cr) + err = needsTransfer(driver, imgClient, imgName, cr) transferFinished <- true }() @@ -137,9 +137,9 @@ func timedNeedsTransfer(imgClient *client.Client, imgName string, cr cruntime.Ma } // needsTransfer returns an error if an image needs to be retransfered -func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager) error { - imgDgst := "" // for instance sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed - if cr.Name() == "docker" && imgClient != nil { // if possible try to get img digest from Client lib which is 4s faster. +func needsTransfer(driver string, imgClient *client.Client, imgName string, cr cruntime.Manager) error { + imgDgst := "" // for instance sha256:7c92a2c6bbcb6b6beff92d0a940779769c2477b807c202954c537e2e0deb9bed + if driver == "docker" && imgClient != nil { // if possible try to get img digest from Client lib which is 4s faster. imgDgst = image.DigestByDockerLib(imgClient, imgName) if imgDgst != "" { if !cr.ImageExists(imgName, imgDgst) { @@ -148,7 +148,7 @@ func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager return nil } } - if cr.Name() == "podman" { + if driver == "podman" { imgDgst = image.DigestByPodmanExec(imgName) if imgDgst != "" { if !cr.ImageExists(imgName, imgDgst) { @@ -158,7 +158,7 @@ func needsTransfer(imgClient *client.Client, imgName string, cr cruntime.Manager } } // if not found with method above try go-container lib (which is 4s slower) - imgDgst = image.DigestByGoLib(cr.Name(), imgName) + imgDgst = image.DigestByGoLib(driver, imgName) if imgDgst == "" { return fmt.Errorf("got empty img digest %q for %s", imgDgst, imgName) }