Add command to save images from the cluster

This is the opposite command of "minikube image load",
and can be used after doing a "minikube image build".

The default is to save images in the cache, but it is
also possible to save to files or to standard output.

cmd/minikube/cmd/image.go

@@ -144,6 +144,77 @@ var loadImageCmd = &cobra.Command{
 	},
 }
 
+func readFile(w io.Writer, tmp string) error {
+	r, err := os.Open(tmp)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(w, r)
+	if err != nil {
+		return err
+	}
+	err = r.Close()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// saveImageCmd represents the image load command
+var saveImageCmd = &cobra.Command{
+	Use:     "save IMAGE [ARCHIVE | -]",
+	Short:   "Save a image from minikube",
+	Long:    "Save a image from minikube",
+	Example: "minikube image save image\nminikube image save image image.tar",
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) == 0 {
+			exit.Message(reason.Usage, "Please provide an image in the container runtime to save from minikube via <minikube image save IMAGE_NAME>")
+		}
+		// Save images from container runtime
+		profile, err := config.LoadProfile(viper.GetString(config.ProfileName))
+		if err != nil {
+			exit.Error(reason.Usage, "loading profile", err)
+		}
+
+		if len(args) > 1 {
+			output = args[1]
+
+			if args[1] == "-" {
+				tmp, err := ioutil.TempFile("", "image.*.tar")
+				if err != nil {
+					exit.Error(reason.GuestImageSave, "Failed to get temp", err)
+				}
+				tmp.Close()
+				output = tmp.Name()
+			}
+
+			if err := machine.DoSaveImages([]string{args[0]}, output, []*config.Profile{profile}, ""); err != nil {
+				exit.Error(reason.GuestImageSave, "Failed to save image", err)
+			}
+
+			if args[1] == "-" {
+				err := readFile(os.Stdout, output)
+				if err != nil {
+					exit.Error(reason.GuestImageSave, "Failed to read temp", err)
+				}
+				os.Remove(output)
+			}
+		} else {
+			if err := machine.SaveAndCacheImages([]string{args[0]}, []*config.Profile{profile}); err != nil {
+				exit.Error(reason.GuestImageSave, "Failed to save image", err)
+			}
+			if imgDaemon || imgRemote {
+				image.UseDaemon(imgDaemon)
+				image.UseRemote(imgRemote)
+				err := image.UploadCachedImage(args[0])
+				if err != nil {
+					exit.Error(reason.GuestImageSave, "Failed to save image", err)
+				}
+			}
+		}
+	},
+}
+
 var removeImageCmd = &cobra.Command{
 	Use:   "rm IMAGE [IMAGE...]",
 	Short: "Remove one or more images",
@@ -258,5 +329,8 @@ func init() {
 	buildImageCmd.Flags().StringArrayVar(&buildEnv, "build-env", nil, "Environment variables to pass to the build. (format: key=value)")
 	buildImageCmd.Flags().StringArrayVar(&buildOpt, "build-opt", nil, "Specify arbitrary flags to pass to the build. (format: key=value)")
 	imageCmd.AddCommand(buildImageCmd)
+	saveImageCmd.Flags().BoolVar(&imgDaemon, "daemon", false, "Cache image to docker daemon")
+	saveImageCmd.Flags().BoolVar(&imgRemote, "remote", false, "Cache image to remote registry")
+	imageCmd.AddCommand(saveImageCmd)
 	imageCmd.AddCommand(listImageCmd)
 }

pkg/minikube/image/image.go

@@ -33,10 +33,12 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/daemon"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/tarball"
 
 	"github.com/pkg/errors"
 	"k8s.io/klog/v2"
 	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/localpath"
 )
 
 const (
@@ -191,6 +193,62 @@ func retrieveRemote(ref name.Reference, p v1.Platform) (v1.Image, error) {
 	return img, err
 }
 
+// imagePathInCache returns path in local cache directory
+func imagePathInCache(img string) string {
+	f := filepath.Join(constants.ImageCacheDir, img)
+	f = localpath.SanitizeCacheDir(f)
+	return f
+}
+
+func UploadCachedImage(imgName string) error {
+	tag, err := name.NewTag(imgName, name.WeakValidation)
+	if err != nil {
+		klog.Infof("error parsing image name %s tag %v ", imgName, err)
+		return err
+	}
+	return uploadImage(tag, imagePathInCache(imgName))
+}
+
+func uploadImage(tag name.Tag, p string) error {
+	var err error
+	var img v1.Image
+
+	if !useDaemon && !useRemote {
+		return fmt.Errorf("neither daemon nor remote")
+	}
+
+	img, err = tarball.ImageFromPath(p, &tag)
+	if err != nil {
+		return errors.Wrap(err, "tarball")
+	}
+	ref := name.Reference(tag)
+
+	klog.Infof("uploading image: %+v from: %s", ref, p)
+	if useDaemon {
+		return uploadDaemon(ref, img)
+	}
+	if useRemote {
+		return uploadRemote(ref, img, defaultPlatform)
+	}
+	return nil
+}
+
+func uploadDaemon(ref name.Reference, img v1.Image) error {
+	resp, err := daemon.Write(ref, img)
+	if err != nil {
+		klog.Warningf("daemon load for %s: %v\n%s", ref, err, resp)
+	}
+	return err
+}
+
+func uploadRemote(ref name.Reference, img v1.Image, p v1.Platform) error {
+	err := remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithPlatform(p))
+	if err != nil {
+		klog.Warningf("remote push for %s: %v", ref, err)
+	}
+	return err
+}
+
 // See https://github.com/kubernetes/minikube/issues/10402
 // check if downloaded image Architecture field matches the requested and fix it otherwise
 func fixPlatform(ref name.Reference, img v1.Image, p v1.Platform) (v1.Image, error) {

pkg/minikube/machine/cache_images.go

@@ -19,6 +19,7 @@ package machine
 import (
 	"fmt"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"sort"
@@ -48,6 +49,9 @@ var loadRoot = path.Join(vmpath.GuestPersistentDir, "images")
 // loadImageLock is used to serialize image loads to avoid overloading the guest VM
 var loadImageLock sync.Mutex
 
+// saveRoot is where images should be saved from within the guest VM
+var saveRoot = path.Join(vmpath.GuestPersistentDir, "images")
+
 // CacheImagesForBootstrapper will cache images for a bootstrapper
 func CacheImagesForBootstrapper(imageRepository string, version string, clusterBootstrapper string) error {
 	images, err := bootstrapper.GetCachedImageList(imageRepository, version, clusterBootstrapper)
@@ -309,6 +313,173 @@ func transferAndLoadImage(cr command.Runner, k8s config.KubernetesConfig, src st
 	return nil
 }
 
+// SaveCachedImages saves from the container runtime to the cache
+func SaveCachedImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error {
+	klog.Infof("SaveImages start: %s", images)
+	start := time.Now()
+
+	defer func() {
+		klog.Infof("SaveImages completed in %s", time.Since(start))
+	}()
+
+	var g errgroup.Group
+
+	for _, image := range images {
+		image := image
+		g.Go(func() error {
+			return transferAndSaveCachedImage(runner, cc.KubernetesConfig, image, cacheDir)
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return errors.Wrap(err, "saving cached images")
+	}
+	klog.Infoln("Successfully saved all cached images")
+	return nil
+}
+
+// SaveLocalImages saves images from the container runtime
+func SaveLocalImages(cc *config.ClusterConfig, runner command.Runner, images []string, output string) error {
+	var g errgroup.Group
+	for _, image := range images {
+		image := image
+		g.Go(func() error {
+			return transferAndSaveImage(runner, cc.KubernetesConfig, output, image)
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return errors.Wrap(err, "saving images")
+	}
+	klog.Infoln("Successfully saved all images")
+	return nil
+}
+
+// SaveAndCacheImages saves images from all profiles into the cache
+func SaveAndCacheImages(images []string, profiles []*config.Profile) error {
+	if len(images) == 0 {
+		return nil
+	}
+
+	return DoSaveImages(images, "", profiles, constants.ImageCacheDir)
+}
+
+// DoSaveImages saves images from all profiles
+func DoSaveImages(images []string, output string, profiles []*config.Profile, cacheDir string) error {
+	api, err := NewAPIClient()
+	if err != nil {
+		return errors.Wrap(err, "api")
+	}
+	defer api.Close()
+
+	klog.Infof("Save images: %q", images)
+
+	succeeded := []string{}
+	failed := []string{}
+
+	for _, p := range profiles { // loading images to all running profiles
+		pName := p.Name // capture the loop variable
+
+		c, err := config.Load(pName)
+		if err != nil {
+			// Non-fatal because it may race with profile deletion
+			klog.Errorf("Failed to load profile %q: %v", pName, err)
+			failed = append(failed, pName)
+			continue
+		}
+
+		for _, n := range c.Nodes {
+			m := config.MachineName(*c, n)
+
+			status, err := Status(api, m)
+			if err != nil {
+				klog.Warningf("error getting status for %s: %v", m, err)
+				failed = append(failed, m)
+				continue
+			}
+
+			if status == state.Running.String() { // the not running hosts will load on next start
+				h, err := api.Load(m)
+				if err != nil {
+					klog.Warningf("Failed to load machine %q: %v", m, err)
+					failed = append(failed, m)
+					continue
+				}
+				cr, err := CommandRunner(h)
+				if err != nil {
+					return err
+				}
+				if cacheDir != "" {
+					// saving image names, to cache
+					err = SaveCachedImages(c, cr, images, cacheDir)
+				} else {
+					// saving mage files
+					err = SaveLocalImages(c, cr, images, output)
+				}
+				if err != nil {
+					failed = append(failed, m)
+					klog.Warningf("Failed to load cached images for profile %s. make sure the profile is running. %v", pName, err)
+					continue
+				}
+				succeeded = append(succeeded, m)
+			}
+		}
+	}
+
+	klog.Infof("succeeded pulling from : %s", strings.Join(succeeded, " "))
+	klog.Infof("failed pulling from : %s", strings.Join(failed, " "))
+	// Live pushes are not considered a failure
+	return nil
+}
+
+// transferAndSaveCachedImage transfers and loads a single image from the cache
+func transferAndSaveCachedImage(cr command.Runner, k8s config.KubernetesConfig, imgName string, cacheDir string) error {
+	dst := filepath.Join(cacheDir, imgName)
+	dst = localpath.SanitizeCacheDir(dst)
+	return transferAndSaveImage(cr, k8s, dst, imgName)
+}
+
+// transferAndSaveImage transfers and loads a single image
+func transferAndSaveImage(cr command.Runner, k8s config.KubernetesConfig, dst string, imgName string) error {
+	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: cr})
+	if err != nil {
+		return errors.Wrap(err, "runtime")
+	}
+
+	klog.Infof("Saving image to: %s", dst)
+	filename := filepath.Base(dst)
+
+	_, err = os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0777)
+	if err != nil {
+		return err
+	}
+
+	f, err := assets.NewFileAsset(dst, saveRoot, filename, "0644")
+	if err != nil {
+		return errors.Wrapf(err, "creating copyable file asset: %s", filename)
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			klog.Warningf("error closing the file %s: %v", f.GetSourcePath(), err)
+		}
+	}()
+
+	src := path.Join(saveRoot, filename)
+	args := append([]string{"rm", "-f"}, src)
+	if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil {
+		return err
+	}
+	err = r.SaveImage(imgName, src)
+	if err != nil {
+		return errors.Wrapf(err, "%s save %s", r.Name(), src)
+	}
+
+	if err := cr.CopyFrom(f); err != nil {
+		return errors.Wrap(err, "transferring cached image")
+	}
+
+	klog.Infof("Transferred and saved %s to cache", dst)
+	return nil
+}
+
 // pullImages pulls images to the container run time
 func pullImages(cruntime cruntime.Manager, images []string) error {
 	klog.Infof("PullImages start: %s", images)

pkg/minikube/reason/reason.go

@@ -317,6 +317,8 @@ var (
 	GuestImageRemove = Kind{ID: "GUEST_IMAGE_REMOVE", ExitCode: ExGuestError}
 	// minikube failed to build an image
 	GuestImageBuild = Kind{ID: "GUEST_IMAGE_BUILD", ExitCode: ExGuestError}
+	// minikube failed to push or save an image
+	GuestImageSave = Kind{ID: "GUEST_IMAGE_SAVE", ExitCode: ExGuestError}
 	// minikube failed to load host
 	GuestLoadHost = Kind{ID: "GUEST_LOAD_HOST", ExitCode: ExGuestError}
 	// minkube failed to create a mount