diff --git a/cmd/minikube/cmd/image.go b/cmd/minikube/cmd/image.go index f76da74e23e5..dbaaf767b3a8 100644 --- a/cmd/minikube/cmd/image.go +++ b/cmd/minikube/cmd/image.go @@ -144,6 +144,77 @@ var loadImageCmd = &cobra.Command{ }, } +func readFile(w io.Writer, tmp string) error { + r, err := os.Open(tmp) + if err != nil { + return err + } + _, err = io.Copy(w, r) + if err != nil { + return err + } + err = r.Close() + if err != nil { + return err + } + return nil +} + +// saveImageCmd represents the image load command +var saveImageCmd = &cobra.Command{ + Use: "save IMAGE [ARCHIVE | -]", + Short: "Save a image from minikube", + Long: "Save a image from minikube", + Example: "minikube image save image\nminikube image save image image.tar", + Run: func(cmd *cobra.Command, args []string) { + if len(args) == 0 { + exit.Message(reason.Usage, "Please provide an image in the container runtime to save from minikube via ") + } + // Save images from container runtime + profile, err := config.LoadProfile(viper.GetString(config.ProfileName)) + if err != nil { + exit.Error(reason.Usage, "loading profile", err) + } + + if len(args) > 1 { + output = args[1] + + if args[1] == "-" { + tmp, err := ioutil.TempFile("", "image.*.tar") + if err != nil { + exit.Error(reason.GuestImageSave, "Failed to get temp", err) + } + tmp.Close() + output = tmp.Name() + } + + if err := machine.DoSaveImages([]string{args[0]}, output, []*config.Profile{profile}, ""); err != nil { + exit.Error(reason.GuestImageSave, "Failed to save image", err) + } + + if args[1] == "-" { + err := readFile(os.Stdout, output) + if err != nil { + exit.Error(reason.GuestImageSave, "Failed to read temp", err) + } + os.Remove(output) + } + } else { + if err := machine.SaveAndCacheImages([]string{args[0]}, []*config.Profile{profile}); err != nil { + exit.Error(reason.GuestImageSave, "Failed to save image", err) + } + if imgDaemon || imgRemote { + image.UseDaemon(imgDaemon) + image.UseRemote(imgRemote) + err := image.UploadCachedImage(args[0]) + if err != nil { + exit.Error(reason.GuestImageSave, "Failed to save image", err) + } + } + } + }, +} + var removeImageCmd = &cobra.Command{ Use: "rm IMAGE [IMAGE...]", Short: "Remove one or more images", @@ -317,6 +388,9 @@ func init() { buildImageCmd.Flags().StringArrayVar(&buildEnv, "build-env", nil, "Environment variables to pass to the build. (format: key=value)") buildImageCmd.Flags().StringArrayVar(&buildOpt, "build-opt", nil, "Specify arbitrary flags to pass to the build. (format: key=value)") imageCmd.AddCommand(buildImageCmd) + saveImageCmd.Flags().BoolVar(&imgDaemon, "daemon", false, "Cache image to docker daemon") + saveImageCmd.Flags().BoolVar(&imgRemote, "remote", false, "Cache image to remote registry") + imageCmd.AddCommand(saveImageCmd) imageCmd.AddCommand(listImageCmd) imageCmd.AddCommand(tagImageCmd) imageCmd.AddCommand(pushImageCmd) diff --git a/pkg/minikube/assets/vm_assets.go b/pkg/minikube/assets/vm_assets.go index b6ec89e9b89f..752953e9606c 100644 --- a/pkg/minikube/assets/vm_assets.go +++ b/pkg/minikube/assets/vm_assets.go @@ -24,6 +24,7 @@ import ( "io" "os" "path" + "strconv" "time" "github.com/pkg/errors" @@ -37,8 +38,11 @@ const MemorySource = "memory" // CopyableFile is something that can be copied type CopyableFile interface { io.Reader + io.Writer GetLength() int + SetLength(int) GetSourcePath() string + GetTargetPath() string GetTargetDir() string GetTargetName() string @@ -62,6 +66,11 @@ func (b *BaseAsset) GetSourcePath() string { return b.SourcePath } +// GetTargetPath returns target path +func (b *BaseAsset) GetTargetPath() string { + return path.Join(b.GetTargetDir(), b.GetTargetName()) +} + // GetTargetDir returns target dir func (b *BaseAsset) GetTargetDir() string { return b.TargetDir @@ -86,6 +95,7 @@ func (b *BaseAsset) GetModTime() (time.Time, error) { type FileAsset struct { BaseAsset reader io.ReadSeeker + writer io.Writer file *os.File // Optional pointer to close file through FileAsset.Close() } @@ -134,6 +144,14 @@ func (f *FileAsset) GetLength() (flen int) { return int(fi.Size()) } +// SetLength sets the file length +func (f *FileAsset) SetLength(flen int) { + err := os.Truncate(f.SourcePath, int64(flen)) + if err != nil { + klog.Errorf("truncate(%q) failed: %v", f.SourcePath, err) + } +} + // GetModTime returns modification time of the file func (f *FileAsset) GetModTime() (time.Time, error) { fi, err := os.Stat(f.SourcePath) @@ -152,6 +170,23 @@ func (f *FileAsset) Read(p []byte) (int, error) { return f.reader.Read(p) } +// Write writes the asset +func (f *FileAsset) Write(p []byte) (int, error) { + if f.writer == nil { + f.file.Close() + perms, err := strconv.ParseUint(f.Permissions, 8, 32) + if err != nil || perms > 07777 { + return 0, err + } + f.file, err = os.OpenFile(f.SourcePath, os.O_RDWR|os.O_CREATE, os.FileMode(perms)) + if err != nil { + return 0, err + } + f.writer = io.Writer(f.file) + } + return f.writer.Write(p) +} + // Seek resets the reader to offset func (f *FileAsset) Seek(offset int64, whence int) (int64, error) { return f.reader.Seek(offset, whence) @@ -177,11 +212,23 @@ func (m *MemoryAsset) GetLength() int { return m.length } +// SetLength returns length +func (m *MemoryAsset) SetLength(len int) { + m.length = len +} + // Read reads the asset func (m *MemoryAsset) Read(p []byte) (int, error) { return m.reader.Read(p) } +// Writer writes the asset +func (m *MemoryAsset) Write(p []byte) (int, error) { + m.length = len(p) + m.reader = bytes.NewReader(p) + return len(p), nil +} + // Seek resets the reader to offset func (m *MemoryAsset) Seek(offset int64, whence int) (int64, error) { return m.reader.Seek(offset, whence) @@ -298,6 +345,11 @@ func (m *BinAsset) GetLength() int { return m.length } +// SetLength sets length +func (m *BinAsset) SetLength(len int) { + m.length = len +} + // Read reads the asset func (m *BinAsset) Read(p []byte) (int, error) { if m.GetLength() == 0 { @@ -306,6 +358,13 @@ func (m *BinAsset) Read(p []byte) (int, error) { return m.reader.Read(p) } +// Write writes the asset +func (m *BinAsset) Write(p []byte) (int, error) { + m.length = len(p) + m.reader = bytes.NewReader(p) + return len(p), nil +} + // Seek resets the reader to offset func (m *BinAsset) Seek(offset int64, whence int) (int64, error) { return m.reader.Seek(offset, whence) diff --git a/pkg/minikube/command/command_runner.go b/pkg/minikube/command/command_runner.go index 41619b1af84f..3abd0dbdfc4b 100644 --- a/pkg/minikube/command/command_runner.go +++ b/pkg/minikube/command/command_runner.go @@ -75,6 +75,9 @@ type Runner interface { // Copy is a convenience method that runs a command to copy a file Copy(assets.CopyableFile) error + // CopyFrom is a convenience method that runs a command to copy a file back + CopyFrom(assets.CopyableFile) error + // Remove is a convenience method that runs a command to remove a file Remove(assets.CopyableFile) error } diff --git a/pkg/minikube/command/exec_runner.go b/pkg/minikube/command/exec_runner.go index b803b9b94924..7d9151977691 100644 --- a/pkg/minikube/command/exec_runner.go +++ b/pkg/minikube/command/exec_runner.go @@ -184,6 +184,24 @@ func (e *execRunner) Copy(f assets.CopyableFile) error { return writeFile(dst, f, os.FileMode(perms)) } +// CopyFrom copies a file +func (e *execRunner) CopyFrom(f assets.CopyableFile) error { + src := path.Join(f.GetTargetDir(), f.GetTargetName()) + + dst := f.GetSourcePath() + klog.Infof("cp: %s --> %s (%d bytes)", src, dst, f.GetLength()) + if f.GetLength() == 0 { + klog.Warningf("0 byte asset: %+v", f) + } + + perms, err := strconv.ParseInt(f.GetPermissions(), 8, 0) + if err != nil || perms > 07777 { + return errors.Wrapf(err, "error converting permissions %s to integer", f.GetPermissions()) + } + + return writeFile(dst, f, os.FileMode(perms)) +} + // Remove removes a file func (e *execRunner) Remove(f assets.CopyableFile) error { dst := filepath.Join(f.GetTargetDir(), f.GetTargetName()) diff --git a/pkg/minikube/command/fake_runner.go b/pkg/minikube/command/fake_runner.go index b663ff7a38fe..0e8521a99f0f 100644 --- a/pkg/minikube/command/fake_runner.go +++ b/pkg/minikube/command/fake_runner.go @@ -142,6 +142,19 @@ func (f *FakeCommandRunner) Copy(file assets.CopyableFile) error { return nil } +func (f *FakeCommandRunner) CopyFrom(file assets.CopyableFile) error { + v, ok := f.fileMap.Load(file.GetSourcePath()) + if !ok { + return fmt.Errorf("not found in map") + } + b := v.(bytes.Buffer) + _, err := io.Copy(file, &b) + if err != nil { + return errors.Wrapf(err, "error writing file: %+v", file) + } + return nil +} + // Remove removes the filename, file contents key value pair from the stored map func (f *FakeCommandRunner) Remove(file assets.CopyableFile) error { f.fileMap.Delete(file.GetSourcePath()) diff --git a/pkg/minikube/command/kic_runner.go b/pkg/minikube/command/kic_runner.go index 71e56feb77c3..7156246ff2c7 100644 --- a/pkg/minikube/command/kic_runner.go +++ b/pkg/minikube/command/kic_runner.go @@ -204,6 +204,15 @@ func (k *kicRunner) Copy(f assets.CopyableFile) error { return k.copy(tf.Name(), dst) } +// CopyFrom copies a file +func (k *kicRunner) CopyFrom(f assets.CopyableFile) error { + src := f.GetTargetPath() + dst := f.GetSourcePath() + + klog.Infof("%s (direct): %s --> %s", k.ociBin, src, dst) + return k.copyFrom(src, dst) +} + // tempDirectory returns the directory to use as the temp directory // or an empty string if it should use the os default temp directory. func tempDirectory(isMinikubeSnap bool, isDockerSnap bool) (string, error) { @@ -229,6 +238,14 @@ func (k *kicRunner) copy(src string, dst string) error { return copyToDocker(src, fullDest) } +func (k *kicRunner) copyFrom(src string, dst string) error { + fullSource := fmt.Sprintf("%s:%s", k.nameOrID, src) + if k.ociBin == oci.Podman { + return copyToPodman(fullSource, dst) + } + return copyToDocker(fullSource, dst) +} + func (k *kicRunner) chmod(dst string, perm string) error { _, err := k.RunCmd(exec.Command("sudo", "chmod", perm, dst)) return err diff --git a/pkg/minikube/command/ssh_runner.go b/pkg/minikube/command/ssh_runner.go index fe92bffe006e..d0044f4c7474 100644 --- a/pkg/minikube/command/ssh_runner.go +++ b/pkg/minikube/command/ssh_runner.go @@ -17,11 +17,14 @@ limitations under the License. package command import ( + "bufio" "bytes" "fmt" "io" "os/exec" "path" + "strconv" + "strings" "sync" "time" @@ -373,3 +376,82 @@ func (s *SSHRunner) Copy(f assets.CopyableFile) error { } return g.Wait() } + +// CopyFrom copies a file from the remote over SSH. +func (s *SSHRunner) CopyFrom(f assets.CopyableFile) error { + dst := path.Join(path.Join(f.GetTargetDir(), f.GetTargetName())) + + sess, err := s.session() + if err != nil { + return errors.Wrap(err, "NewSession") + } + defer func() { + if err := sess.Close(); err != nil { + if err != io.EOF { + klog.Errorf("session close: %v", err) + } + } + }() + + cmd := exec.Command("stat", "-c", "%s", dst) + rr, err := s.RunCmd(cmd) + if err != nil { + return fmt.Errorf("%s: %v", cmd, err) + } + length, err := strconv.Atoi(strings.TrimSuffix(rr.Stdout.String(), "\n")) + if err != nil { + return err + } + src := f.GetSourcePath() + klog.Infof("scp %s --> %s (%d bytes)", dst, src, length) + f.SetLength(length) + + r, err := sess.StdoutPipe() + if err != nil { + return errors.Wrap(err, "StdoutPipe") + } + w, err := sess.StdinPipe() + if err != nil { + return errors.Wrap(err, "StdinPipe") + } + // The scpcmd below *should not* return until all data is copied and the + // StdinPipe is closed. But let's use errgroup to make it explicit. + var g errgroup.Group + var copied int64 + + g.Go(func() error { + defer w.Close() + br := bufio.NewReader(r) + fmt.Fprint(w, "\x00") + b, err := br.ReadBytes('\n') + if err != nil { + return errors.Wrap(err, "ReadBytes") + } + if b[0] != 'C' { + return fmt.Errorf("unexpected: %v", b) + } + fmt.Fprint(w, "\x00") + + copied = 0 + for copied < int64(length) { + n, err := io.CopyN(f, br, int64(length)) + if err != nil { + return errors.Wrap(err, "io.CopyN") + } + copied += n + } + fmt.Fprint(w, "\x00") + err = sess.Wait() + if err != nil { + return err + } + return nil + }) + + scp := fmt.Sprintf("sudo scp -f %s", f.GetTargetPath()) + err = sess.Start(scp) + if err != nil { + return fmt.Errorf("%s: %s", scp, err) + } + return g.Wait() +} diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 71ca9a90489b..aba79f668f8e 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -248,10 +248,14 @@ func (r *Containerd) Disable() error { return r.Init.ForceStop("containerd") } -// ImageExists checks if an image exists, expected input format +// ImageExists checks if image exists based on image name and optionally image sha func (r *Containerd) ImageExists(name string, sha string) bool { - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo ctr -n=k8s.io images check | grep %s | grep %s", name, sha)) - if _, err := r.Runner.RunCmd(c); err != nil { + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo ctr -n=k8s.io images check | grep %s", name)) + rr, err := r.Runner.RunCmd(c) + if err != nil { + return false + } + if sha != "" && !strings.Contains(rr.Output(), sha) { return false } return true diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 8d69afa1e0f0..1bca1c5eeb95 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -162,7 +162,7 @@ func (r *CRIO) Disable() error { return r.Init.ForceStop("crio") } -// ImageExists checks if an image exists +// ImageExists checks if image exists based on image name and optionally image sha func (r *CRIO) ImageExists(name string, sha string) bool { // expected output looks like [NAME@sha256:SHA] c := exec.Command("sudo", "podman", "image", "inspect", "--format", "{{.Id}}", name) @@ -170,7 +170,7 @@ func (r *CRIO) ImageExists(name string, sha string) bool { if err != nil { return false } - if !strings.Contains(rr.Output(), sha) { + if sha != "" && !strings.Contains(rr.Output(), sha) { return false } return true diff --git a/pkg/minikube/cruntime/cruntime.go b/pkg/minikube/cruntime/cruntime.go index 1ca58e01d42c..d736d809ce6b 100644 --- a/pkg/minikube/cruntime/cruntime.go +++ b/pkg/minikube/cruntime/cruntime.go @@ -65,6 +65,8 @@ type CommandRunner interface { WaitCmd(sc *command.StartedCmd) (*command.RunResult, error) // Copy is a convenience method that runs a command to copy a file Copy(assets.CopyableFile) error + // CopyFrom is a convenience method that runs a command to copy a file back + CopyFrom(assets.CopyableFile) error // Remove is a convenience method that runs a command to remove a file Remove(assets.CopyableFile) error } @@ -106,7 +108,7 @@ type Manager interface { // Push an image from the runtime to the container registry PushImage(string) error - // ImageExists takes image name and image sha checks if an it exists + // ImageExists takes image name and optionally image sha to check if an image exists ImageExists(string, string) bool // ListImages returns a list of images managed by this container runtime ListImages(ListImagesOptions) ([]string, error) diff --git a/pkg/minikube/cruntime/cruntime_test.go b/pkg/minikube/cruntime/cruntime_test.go index ca1c6cd54b37..3a55059cc4fe 100644 --- a/pkg/minikube/cruntime/cruntime_test.go +++ b/pkg/minikube/cruntime/cruntime_test.go @@ -236,6 +236,10 @@ func (f *FakeRunner) Copy(assets.CopyableFile) error { return nil } +func (f *FakeRunner) CopyFrom(assets.CopyableFile) error { + return nil +} + func (f *FakeRunner) Remove(assets.CopyableFile) error { return nil } diff --git a/pkg/minikube/cruntime/docker.go b/pkg/minikube/cruntime/docker.go index 2b06321f431e..d3de483deee6 100644 --- a/pkg/minikube/cruntime/docker.go +++ b/pkg/minikube/cruntime/docker.go @@ -165,7 +165,7 @@ func (r *Docker) Disable() error { return r.Init.Mask("docker.service") } -// ImageExists checks if an image exists +// ImageExists checks if image exists based on image name and optionally image sha func (r *Docker) ImageExists(name string, sha string) bool { // expected output looks like [SHA_ALGO:SHA] c := exec.Command("docker", "image", "inspect", "--format", "{{.Id}}", name) @@ -173,7 +173,7 @@ func (r *Docker) ImageExists(name string, sha string) bool { if err != nil { return false } - if !strings.Contains(rr.Output(), sha) { + if sha != "" && !strings.Contains(rr.Output(), sha) { return false } return true @@ -201,7 +201,7 @@ func (r *Docker) ListImages(ListImagesOptions) ([]string, error) { // LoadImage loads an image into this runtime func (r *Docker) LoadImage(path string) error { klog.Infof("Loading image: %s", path) - c := exec.Command("docker", "load", "-i", path) + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo cat %s | docker load", path)) if _, err := r.Runner.RunCmd(c); err != nil { return errors.Wrap(err, "loadimage docker.") } @@ -224,7 +224,7 @@ func (r *Docker) PullImage(name string) error { // SaveImage saves an image from this runtime func (r *Docker) SaveImage(name string, path string) error { klog.Infof("Saving image %s: %s", name, path) - c := exec.Command("docker", "save", name, "-o", path) + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("docker save '%s' | sudo tee %s >/dev/null", name, path)) if _, err := r.Runner.RunCmd(c); err != nil { return errors.Wrap(err, "saveimage docker.") } diff --git a/pkg/minikube/image/image.go b/pkg/minikube/image/image.go index 7814ce9abdda..2bbcf919cbf3 100644 --- a/pkg/minikube/image/image.go +++ b/pkg/minikube/image/image.go @@ -33,10 +33,12 @@ import ( "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/pkg/errors" "k8s.io/klog/v2" "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/localpath" ) const ( @@ -191,6 +193,62 @@ func retrieveRemote(ref name.Reference, p v1.Platform) (v1.Image, error) { return img, err } +// imagePathInCache returns path in local cache directory +func imagePathInCache(img string) string { + f := filepath.Join(constants.ImageCacheDir, img) + f = localpath.SanitizeCacheDir(f) + return f +} + +func UploadCachedImage(imgName string) error { + tag, err := name.NewTag(imgName, name.WeakValidation) + if err != nil { + klog.Infof("error parsing image name %s tag %v ", imgName, err) + return err + } + return uploadImage(tag, imagePathInCache(imgName)) +} + +func uploadImage(tag name.Tag, p string) error { + var err error + var img v1.Image + + if !useDaemon && !useRemote { + return fmt.Errorf("neither daemon nor remote") + } + + img, err = tarball.ImageFromPath(p, &tag) + if err != nil { + return errors.Wrap(err, "tarball") + } + ref := name.Reference(tag) + + klog.Infof("uploading image: %+v from: %s", ref, p) + if useDaemon { + return uploadDaemon(tag, img) + } + if useRemote { + return uploadRemote(ref, img, defaultPlatform) + } + return nil +} + +func uploadDaemon(tag name.Tag, img v1.Image) error { + resp, err := daemon.Write(tag, img) + if err != nil { + klog.Warningf("daemon load for %s: %v\n%s", tag, err, resp) + } + return err +} + +func uploadRemote(ref name.Reference, img v1.Image, p v1.Platform) error { + err := remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithPlatform(p)) + if err != nil { + klog.Warningf("remote push for %s: %v", ref, err) + } + return err +} + // See https://github.com/kubernetes/minikube/issues/10402 // check if downloaded image Architecture field matches the requested and fix it otherwise func fixPlatform(ref name.Reference, img v1.Image, p v1.Platform) (v1.Image, error) { diff --git a/pkg/minikube/machine/cache_images.go b/pkg/minikube/machine/cache_images.go index f2f03dc64b58..24928a925663 100644 --- a/pkg/minikube/machine/cache_images.go +++ b/pkg/minikube/machine/cache_images.go @@ -19,6 +19,7 @@ package machine import ( "fmt" "os" + "os/exec" "path" "path/filepath" "sort" @@ -48,6 +49,9 @@ var loadRoot = path.Join(vmpath.GuestPersistentDir, "images") // loadImageLock is used to serialize image loads to avoid overloading the guest VM var loadImageLock sync.Mutex +// saveRoot is where images should be saved from within the guest VM +var saveRoot = path.Join(vmpath.GuestPersistentDir, "images") + // CacheImagesForBootstrapper will cache images for a bootstrapper func CacheImagesForBootstrapper(imageRepository string, version string, clusterBootstrapper string) error { images, err := bootstrapper.GetCachedImageList(imageRepository, version, clusterBootstrapper) @@ -326,6 +330,177 @@ func removeExistingImage(r cruntime.Manager, src string, imgName string) error { return nil } +// SaveCachedImages saves from the container runtime to the cache +func SaveCachedImages(cc *config.ClusterConfig, runner command.Runner, images []string, cacheDir string) error { + klog.Infof("SaveImages start: %s", images) + start := time.Now() + + defer func() { + klog.Infof("SaveImages completed in %s", time.Since(start)) + }() + + var g errgroup.Group + + for _, image := range images { + image := image + g.Go(func() error { + return transferAndSaveCachedImage(runner, cc.KubernetesConfig, image, cacheDir) + }) + } + if err := g.Wait(); err != nil { + return errors.Wrap(err, "saving cached images") + } + klog.Infoln("Successfully saved all cached images") + return nil +} + +// SaveLocalImages saves images from the container runtime +func SaveLocalImages(cc *config.ClusterConfig, runner command.Runner, images []string, output string) error { + var g errgroup.Group + for _, image := range images { + image := image + g.Go(func() error { + return transferAndSaveImage(runner, cc.KubernetesConfig, output, image) + }) + } + if err := g.Wait(); err != nil { + return errors.Wrap(err, "saving images") + } + klog.Infoln("Successfully saved all images") + return nil +} + +// SaveAndCacheImages saves images from all profiles into the cache +func SaveAndCacheImages(images []string, profiles []*config.Profile) error { + if len(images) == 0 { + return nil + } + + return DoSaveImages(images, "", profiles, constants.ImageCacheDir) +} + +// DoSaveImages saves images from all profiles +func DoSaveImages(images []string, output string, profiles []*config.Profile, cacheDir string) error { + api, err := NewAPIClient() + if err != nil { + return errors.Wrap(err, "api") + } + defer api.Close() + + klog.Infof("Save images: %q", images) + + succeeded := []string{} + failed := []string{} + + for _, p := range profiles { // loading images to all running profiles + pName := p.Name // capture the loop variable + + c, err := config.Load(pName) + if err != nil { + // Non-fatal because it may race with profile deletion + klog.Errorf("Failed to load profile %q: %v", pName, err) + failed = append(failed, pName) + continue + } + + for _, n := range c.Nodes { + m := config.MachineName(*c, n) + + status, err := Status(api, m) + if err != nil { + klog.Warningf("error getting status for %s: %v", m, err) + failed = append(failed, m) + continue + } + + if status == state.Running.String() { // the not running hosts will load on next start + h, err := api.Load(m) + if err != nil { + klog.Warningf("Failed to load machine %q: %v", m, err) + failed = append(failed, m) + continue + } + cr, err := CommandRunner(h) + if err != nil { + return err + } + if cacheDir != "" { + // saving image names, to cache + err = SaveCachedImages(c, cr, images, cacheDir) + } else { + // saving mage files + err = SaveLocalImages(c, cr, images, output) + } + if err != nil { + failed = append(failed, m) + klog.Warningf("Failed to load cached images for profile %s. make sure the profile is running. %v", pName, err) + continue + } + succeeded = append(succeeded, m) + } + } + } + + klog.Infof("succeeded pulling from : %s", strings.Join(succeeded, " ")) + klog.Infof("failed pulling from : %s", strings.Join(failed, " ")) + // Live pushes are not considered a failure + return nil +} + +// transferAndSaveCachedImage transfers and loads a single image from the cache +func transferAndSaveCachedImage(cr command.Runner, k8s config.KubernetesConfig, imgName string, cacheDir string) error { + dst := filepath.Join(cacheDir, imgName) + dst = localpath.SanitizeCacheDir(dst) + return transferAndSaveImage(cr, k8s, dst, imgName) +} + +// transferAndSaveImage transfers and loads a single image +func transferAndSaveImage(cr command.Runner, k8s config.KubernetesConfig, dst string, imgName string) error { + r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime, Runner: cr}) + if err != nil { + return errors.Wrap(err, "runtime") + } + + if !r.ImageExists(imgName, "") { + return errors.Errorf("image %s not found", imgName) + } + + klog.Infof("Saving image to: %s", dst) + filename := filepath.Base(dst) + + _, err = os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0777) + if err != nil { + return err + } + + f, err := assets.NewFileAsset(dst, saveRoot, filename, "0644") + if err != nil { + return errors.Wrapf(err, "creating copyable file asset: %s", filename) + } + defer func() { + if err := f.Close(); err != nil { + klog.Warningf("error closing the file %s: %v", f.GetSourcePath(), err) + } + }() + + src := path.Join(saveRoot, filename) + args := append([]string{"rm", "-f"}, src) + if _, err := cr.RunCmd(exec.Command("sudo", args...)); err != nil { + return err + } + err = r.SaveImage(imgName, src) + if err != nil { + return errors.Wrapf(err, "%s save %s", r.Name(), src) + } + + if err := cr.CopyFrom(f); err != nil { + return errors.Wrap(err, "transferring cached image") + } + + klog.Infof("Transferred and saved %s to cache", dst) + return nil +} + // pullImages pulls images to the container run time func pullImages(cruntime cruntime.Manager, images []string) error { klog.Infof("PullImages start: %s", images) diff --git a/pkg/minikube/reason/reason.go b/pkg/minikube/reason/reason.go index a3dd17643230..dba72cdab48c 100644 --- a/pkg/minikube/reason/reason.go +++ b/pkg/minikube/reason/reason.go @@ -317,10 +317,12 @@ var ( GuestImageRemove = Kind{ID: "GUEST_IMAGE_REMOVE", ExitCode: ExGuestError} // minikube failed to pull an image GuestImagePull = Kind{ID: "GUEST_IMAGE_PULL", ExitCode: ExGuestError} - // minikube failed to push an image - GuestImagePush = Kind{ID: "GUEST_IMAGE_PUSH", ExitCode: ExGuestError} // minikube failed to build an image GuestImageBuild = Kind{ID: "GUEST_IMAGE_BUILD", ExitCode: ExGuestError} + // minikube failed to push or save an image + GuestImageSave = Kind{ID: "GUEST_IMAGE_SAVE", ExitCode: ExGuestError} + // minikube failed to push an image + GuestImagePush = Kind{ID: "GUEST_IMAGE_PUSH", ExitCode: ExGuestError} // minikube failed to tag an image GuestImageTag = Kind{ID: "GUEST_IMAGE_TAG", ExitCode: ExGuestError} // minikube failed to load host diff --git a/site/content/en/docs/commands/image.md b/site/content/en/docs/commands/image.md index 6f6537c5ff64..09df6a14c43b 100644 --- a/site/content/en/docs/commands/image.md +++ b/site/content/en/docs/commands/image.md @@ -348,6 +348,54 @@ $ minikube image unload image busybox --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging ``` +## minikube image save + +Save a image from minikube + +### Synopsis + +Save a image from minikube + +```shell +minikube image save IMAGE [ARCHIVE | -] [flags] +``` + +### Examples + +``` +minikube image save image +minikube image save image image.tar +``` + +### Options + +``` + --daemon Cache image to docker daemon + --remote Cache image to remote registry +``` + +### Options inherited from parent commands + +``` + --add_dir_header If true, adds the file directory to the header of the log messages + --alsologtostderr log to standard error as well as files + -b, --bootstrapper string The name of the cluster bootstrapper that will set up the Kubernetes cluster. (default "kubeadm") + -h, --help + --log_backtrace_at traceLocation when logging hits line file:N, emit a stack trace (default :0) + --log_dir string If non-empty, write log files in this directory + --log_file string If non-empty, use this log file + --log_file_max_size uint Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800) + --logtostderr log to standard error instead of files + --one_output If true, only write logs to their native severity level (vs also writing to each lower severity level) + -p, --profile string The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently. (default "minikube") + --skip_headers If true, avoid header prefixes in the log messages + --skip_log_headers If true, avoid headers when opening log files + --stderrthreshold severity logs at or above this threshold go to stderr (default 2) + --user string Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username. + -v, --v Level number for the log level verbosity + --vmodule moduleSpec comma-separated list of pattern=N settings for file-filtered logging +``` + ## minikube image tag Tag images diff --git a/site/content/en/docs/contrib/errorcodes.en.md b/site/content/en/docs/contrib/errorcodes.en.md index a491beb0b4bb..4b26b4a3ccaa 100644 --- a/site/content/en/docs/contrib/errorcodes.en.md +++ b/site/content/en/docs/contrib/errorcodes.en.md @@ -381,12 +381,15 @@ minikube failed to remove an image "GUEST_IMAGE_PULL" (Exit code ExGuestError) minikube failed to pull an image -"GUEST_IMAGE_PUSH" (Exit code ExGuestError) -minikube failed to push an image - "GUEST_IMAGE_BUILD" (Exit code ExGuestError) minikube failed to build an image +"GUEST_IMAGE_SAVE" (Exit code ExGuestError) +minikube failed to push or save an image + +"GUEST_IMAGE_PUSH" (Exit code ExGuestError) +minikube failed to push an image + "GUEST_IMAGE_TAG" (Exit code ExGuestError) minikube failed to tag an image diff --git a/site/content/en/docs/contrib/tests.en.md b/site/content/en/docs/contrib/tests.en.md index f3c3efe62575..119eef71edac 100644 --- a/site/content/en/docs/contrib/tests.en.md +++ b/site/content/en/docs/contrib/tests.en.md @@ -84,6 +84,12 @@ makes sure that `minikube image load` works from a local file #### validateRemoveImage makes sures that `minikube image rm` works as expected +#### validateSaveImage +makes sure that `minikube image save` works as expected + +#### validateSaveImageToFile +makes sure that `minikube image save` works to a local file + #### validateBuildImage makes sures that `minikube image build` works as expected diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index 10bee2cbbfd7..c758fb8fcaae 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -151,8 +151,10 @@ func TestFunctional(t *testing.T) { {"PodmanEnv", validatePodmanEnv}, {"NodeLabels", validateNodeLabels}, {"LoadImage", validateLoadImage}, + {"SaveImage", validateSaveImage}, {"RemoveImage", validateRemoveImage}, {"LoadImageFromFile", validateLoadImageFromFile}, + {"SaveImageToFile", validateSaveImageToFile}, {"BuildImage", validateBuildImage}, {"ListImages", validateListImages}, {"NonActiveRuntimeDisabled", validateNotActiveRuntimeDisabled}, @@ -206,7 +208,6 @@ func cleanupUnwantedImages(ctx context.Context, t *testing.T, profile string) { } }) } - } // validateNodeLabels checks if minikube cluster is created with correct kubernetes's node label @@ -249,7 +250,7 @@ func validateLoadImage(ctx context.Context, t *testing.T, profile string) { } // try to load the new image into minikube - rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", newImage)) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", "--daemon", newImage)) if err != nil { t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) } @@ -289,7 +290,7 @@ func validateLoadImageFromFile(ctx context.Context, t *testing.T, profile string } // save image to file - imageFile := "busybox.tar" + imageFile := "busybox-load.tar" rr, err = Run(t, exec.CommandContext(ctx, "docker", "save", "-o", imageFile, taggedImage)) if err != nil { t.Fatalf("failed to save image to file: %v\n%s", err, rr.Output()) @@ -302,7 +303,7 @@ func validateLoadImageFromFile(ctx context.Context, t *testing.T, profile string t.Fatalf("failed to get absolute path of file %q: %v", imageFile, err) } rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "load", imagePath)) - if err != nil { + if err != nil || rr.Stderr.String() != "" { t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) } @@ -312,7 +313,7 @@ func validateLoadImageFromFile(ctx context.Context, t *testing.T, profile string t.Fatalf("listing images: %v\n%s", err, rr.Output()) } if !strings.Contains(rr.Output(), tag) { - t.Fatalf("expected %s to be loaded into minikube but the image is not there", taggedImage) + t.Fatalf("expected %s to be loaded into minikube but the image is not there: %v", taggedImage, rr.Output()) } } @@ -363,6 +364,101 @@ func validateRemoveImage(ctx context.Context, t *testing.T, profile string) { } +// validateSaveImage makes sure that `minikube image save` works as expected +func validateSaveImage(ctx context.Context, t *testing.T, profile string) { + if NoneDriver() { + t.Skip("load image not available on none driver") + } + if GithubActionRunner() && runtime.GOOS == "darwin" { + t.Skip("skipping on github actions and darwin, as this test requires a running docker daemon") + } + defer PostMortemLogs(t, profile) + // pull busybox + busyboxImage := "docker.io/library/busybox:1.29" + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "pull", busyboxImage)) + if err != nil { + t.Fatalf("failed to setup test (pull image): %v\n%s", err, rr.Output()) + } + + // tag busybox + name := "busybox" + tag := fmt.Sprintf("save-%s", profile) + newImage := fmt.Sprintf("docker.io/library/%s:%s", name, tag) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "tag", busyboxImage, newImage)) + if err != nil { + t.Fatalf("failed to setup test (tag image) : %v\n%s", err, rr.Output()) + } + + // try to save the new image from minikube + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "save", "--daemon", newImage)) + if err != nil { + t.Fatalf("loading image into minikube: %v\n%s", err, rr.Output()) + } + + // make sure the image was correctly loaded + rr, err = Run(t, exec.CommandContext(ctx, "docker", "images", name)) + if err != nil { + t.Fatalf("listing images: %v\n%s", err, rr.Output()) + } + if !strings.Contains(rr.Output(), fmt.Sprintf("save-%s", profile)) { + t.Fatalf("expected %s to be loaded into minikube but the image is not there", newImage) + } + +} + +// validateSaveImageToFile makes sure that `minikube image save` works to a local file +func validateSaveImageToFile(ctx context.Context, t *testing.T, profile string) { + if NoneDriver() { + t.Skip("save image not available on none driver") + } + if GithubActionRunner() && runtime.GOOS == "darwin" { + t.Skip("skipping on github actions and darwin, as this test requires a running docker daemon") + } + defer PostMortemLogs(t, profile) + // pull busybox + busyboxImage := "docker.io/library/busybox:1.30" + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "pull", busyboxImage)) + if err != nil { + t.Fatalf("failed to setup test (pull image): %v\n%s", err, rr.Output()) + } + + name := "busybox" + tag := fmt.Sprintf("save-to-file-%s", profile) + taggedImage := fmt.Sprintf("docker.io/library/%s:%s", name, tag) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "tag", busyboxImage, taggedImage)) + if err != nil { + t.Fatalf("failed to setup test (tag image) : %v\n%s", err, rr.Output()) + } + + // try to save the new image from minikube + imageFile := "busybox-save.tar" + imagePath, err := filepath.Abs(imageFile) + if err != nil { + t.Fatalf("failed to get absolute path of file %q: %v", imageFile, err) + } + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "image", "save", taggedImage, imagePath)) + if err != nil { + t.Fatalf("saving image from minikube: %v\n%s", err, rr.Output()) + } + + // load image from file + rr, err = Run(t, exec.CommandContext(ctx, "docker", "load", "-i", imagePath)) + if err != nil { + t.Fatalf("failed to load image to file: %v\n%s", err, rr.Output()) + } + defer os.Remove(imageFile) + + // make sure the image was correctly loaded + rr, err = Run(t, exec.CommandContext(ctx, "docker", "images", name)) + if err != nil { + t.Fatalf("listing images: %v\n%s", err, rr.Output()) + } + if !strings.Contains(rr.Output(), tag) { + t.Fatalf("expected %s to be loaded but the image is not there", taggedImage) + } + +} + func inspectImage(ctx context.Context, t *testing.T, profile string, image string) (*RunResult, error) { var cmd *exec.Cmd if ContainerRuntime() == "docker" {