Make crane flatten work with indexes (#1105)

* Expose crane.GetOptions * Add easier accessor for remote index * Support flattening index
google · Aug 23, 2021 · e92a648 · e92a648
1 parent de8aff8
commit e92a648
Show file tree

Hide file tree

Showing 14 changed files with 296 additions and 142 deletions.
diff --git a/cmd/crane/cmd/flatten.go b/cmd/crane/cmd/flatten.go
@@ -21,124 +21,234 @@ import (
 	"log"
 
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/logs"
 	"github.com/google/go-containerregistry/pkg/name"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/partial"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
 	"github.com/google/go-containerregistry/pkg/v1/stream"
 	"github.com/spf13/cobra"
 )
 
 // NewCmdFlatten creates a new cobra.Command for the flatten subcommand.
 func NewCmdFlatten(options *[]crane.Option) *cobra.Command {
-	var newRef string
+	var dst string
 
 	flattenCmd := &cobra.Command{
 		Use:   "flatten",
 		Short: "Flatten an image's layers into a single layer",
 		Args:  cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
+			// We need direct access to the underlying remote options because crane
+			// doesn't expose great facilities for working with an index (yet).
+			o := crane.GetOptions(*options...)
+
 			// Pull image and get config.
-			ref := args[0]
+			src := args[0]
 
 			// If the new ref isn't provided, write over the original image.
 			// If that ref was provided by digest (e.g., output from
 			// another crane command), then strip that and push the
 			// mutated image by digest instead.
-			if newRef == "" {
-				newRef = ref
+			if dst == "" {
+				dst = src
 			}
 
-			// Stupid hack to support insecure flag.
-			nameOpt := []name.Option{}
-			if ok, err := cmd.Parent().PersistentFlags().GetBool("insecure"); err != nil {
-				log.Fatalf("flag problems: %v", err)
-			} else if ok {
-				nameOpt = append(nameOpt, name.Insecure)
-			}
-			r, err := name.ParseReference(newRef, nameOpt...)
+			ref, err := name.ParseReference(src, o.Name...)
 			if err != nil {
-				log.Fatalf("parsing %s: %v", newRef, err)
+				log.Fatalf("parsing %s: %v", src, err)
 			}
-
-			desc, err := crane.Head(ref, *options...)
+			newRef, err := name.ParseReference(dst, o.Name...)
 			if err != nil {
-				log.Fatalf("checking %s: %v", ref, err)
-			}
-			if !cmd.Parent().PersistentFlags().Changed("platform") && desc.MediaType.IsIndex() {
-				log.Fatalf("flattening an index is not yet supported")
+				log.Fatalf("parsing %s: %v", dst, err)
 			}
+			repo := newRef.Context()
 
-			old, err := crane.Pull(ref, *options...)
+			flat, err := flatten(ref, repo, cmd.Parent().Use, o)
 			if err != nil {
-				log.Fatalf("pulling %s: %v", ref, err)
+				log.Fatalf("flattening %s: %v", ref, err)
 			}
 
-			m, err := old.Manifest()
+			digest, err := flat.Digest()
 			if err != nil {
-				log.Fatalf("reading manifest: %v", err)
+				log.Fatalf("digesting new image: %v", err)
 			}
 
-			cf, err := old.ConfigFile()
-			if err != nil {
-				log.Fatalf("getting config: %v", err)
+			if _, ok := ref.(name.Digest); ok {
+				newRef = repo.Digest(digest.String())
 			}
-			cf = cf.DeepCopy()
 
-			oldHistory, err := json.Marshal(cf.History)
-			if err != nil {
-				log.Fatalf("marshal history")
+			if err := push(flat, newRef, o); err != nil {
+				log.Fatalf("pushing %s: %v", newRef, err)
 			}
+			fmt.Println(repo.Digest(digest.String()))
+		},
+	}
+	flattenCmd.Flags().StringVarP(&dst, "tag", "t", "", "New tag to apply to flattened image. If not provided, push by digest to the original image repository.")
+	return flattenCmd
+}
 
-			// Clear layer-specific config file information.
-			cf.RootFS.DiffIDs = []v1.Hash{}
-			cf.History = []v1.History{}
+func flatten(ref name.Reference, repo name.Repository, use string, o crane.Options) (partial.Describable, error) {
+	desc, err := remote.Get(ref, o.Remote...)
+	if err != nil {
+		return nil, fmt.Errorf("pulling %s: %v", ref, err)
+	}
 
-			img, err := mutate.ConfigFile(empty.Image, cf)
-			if err != nil {
-				log.Fatalf("mutating config: %v", err)
-			}
+	if desc.MediaType.IsIndex() {
+		idx, err := desc.ImageIndex()
+		if err != nil {
+			return nil, err
+		}
+		return flattenIndex(idx, repo, use, o)
+	} else if desc.MediaType.IsImage() {
+		img, err := desc.Image()
+		if err != nil {
+			return nil, err
+		}
+		return flattenImage(img, repo, use, o)
+	}
 
-			// TODO: Make compression configurable?
-			layer := stream.NewLayer(mutate.Extract(old), stream.WithCompressionLevel(gzip.BestCompression))
+	return nil, fmt.Errorf("can't flatten %s", desc.MediaType)
+}
 
-			img, err = mutate.Append(img, mutate.Addendum{
-				Layer: layer,
-				History: v1.History{
-					CreatedBy: fmt.Sprintf("%s flatten %s", cmd.Parent().Use, desc.Digest),
-					Comment:   string(oldHistory),
-				},
-			})
-			if err != nil {
-				log.Fatalf("appending layers: %v", err)
-			}
+func push(flat partial.Describable, ref name.Reference, o crane.Options) error {
+	if idx, ok := flat.(v1.ImageIndex); ok {
+		return remote.WriteIndex(ref, idx, o.Remote...)
+	} else if img, ok := flat.(v1.Image); ok {
+		return remote.Write(ref, img, o.Remote...)
+	}
 
-			// Retain any annotations from the original image.
-			if len(m.Annotations) != 0 {
-				img = mutate.Annotations(img, m.Annotations).(v1.Image)
-			}
+	return fmt.Errorf("can't push %T", flat)
+}
 
-			if _, ok := r.(name.Digest); ok {
-				// If we're pushing by digest, we need to upload the layer first.
-				if err := crane.Upload(layer, r.Context().String(), *options...); err != nil {
-					log.Fatalf("uploading layer: %v", err)
-				}
-				digest, err := img.Digest()
-				if err != nil {
-					log.Fatalf("digesting new image: %v", err)
-				}
-				newRef = r.Context().Digest(digest.String()).String()
-			}
-			if err := crane.Push(img, newRef, *options...); err != nil {
-				log.Fatalf("pushing %s: %v", newRef, err)
-			}
-			digest, err := img.Digest()
-			if err != nil {
-				log.Fatalf("digesting new image: %v", err)
-			}
-			fmt.Println(r.Context().Digest(digest.String()))
+type remoteIndex interface {
+	Manifests() ([]partial.Describable, error)
+}
+
+func flattenIndex(old v1.ImageIndex, repo name.Repository, use string, o crane.Options) (partial.Describable, error) {
+	ri, ok := old.(remoteIndex)
+	if !ok {
+		return nil, fmt.Errorf("unexpected index")
+	}
+
+	m, err := old.IndexManifest()
+	if err != nil {
+		return nil, err
+	}
+
+	manifests, err := ri.Manifests()
+	if err != nil {
+		return nil, err
+	}
+
+	adds := []mutate.IndexAddendum{}
+
+	for _, m := range manifests {
+		// Keep the old descriptor (annotations and whatnot).
+		desc, err := partial.Descriptor(m)
+		if err != nil {
+			return nil, err
+		}
+
+		flattened, err := flattenChild(m, repo, use, o)
+		if err != nil {
+			return nil, err
+		}
+		desc.Size, err = flattened.Size()
+		if err != nil {
+			return nil, err
+		}
+		desc.Digest, err = flattened.Digest()
+		if err != nil {
+			return nil, err
+		}
+		adds = append(adds, mutate.IndexAddendum{
+			Add:        flattened,
+			Descriptor: *desc,
+		})
+	}
+
+	idx := mutate.AppendManifests(empty.Index, adds...)
+
+	// Retain any annotations from the original index.
+	if len(m.Annotations) != 0 {
+		idx = mutate.Annotations(idx, m.Annotations).(v1.ImageIndex)
+	}
+
+	// This is stupid, but some registries get mad if you try to push OCI media types that reference docker media types.
+	mt, err := old.MediaType()
+	if err != nil {
+		return nil, err
+	}
+	idx = mutate.IndexMediaType(idx, mt)
+
+	return idx, nil
+}
+
+func flattenChild(old partial.Describable, repo name.Repository, use string, o crane.Options) (partial.Describable, error) {
+	if idx, ok := old.(v1.ImageIndex); ok {
+		return flattenIndex(idx, repo, use, o)
+	} else if img, ok := old.(v1.Image); ok {
+		return flattenImage(img, repo, use, o)
+	}
+
+	logs.Warn.Printf("can't flatten %T, skipping", old)
+	return old, nil
+}
+
+func flattenImage(old v1.Image, repo name.Repository, use string, o crane.Options) (partial.Describable, error) {
+	digest, err := old.Digest()
+	if err != nil {
+		return nil, fmt.Errorf("getting old digest: %v", err)
+	}
+	m, err := old.Manifest()
+	if err != nil {
+		return nil, fmt.Errorf("reading manifest: %v", err)
+	}
+
+	cf, err := old.ConfigFile()
+	if err != nil {
+		return nil, fmt.Errorf("getting config: %v", err)
+	}
+	cf = cf.DeepCopy()
+
+	oldHistory, err := json.Marshal(cf.History)
+	if err != nil {
+		return nil, fmt.Errorf("marshal history")
+	}
+
+	// Clear layer-specific config file information.
+	cf.RootFS.DiffIDs = []v1.Hash{}
+	cf.History = []v1.History{}
+
+	img, err := mutate.ConfigFile(empty.Image, cf)
+	if err != nil {
+		return nil, fmt.Errorf("mutating config: %v", err)
+	}
+
+	// TODO: Make compression configurable?
+	layer := stream.NewLayer(mutate.Extract(old), stream.WithCompressionLevel(gzip.BestCompression))
+	if err := remote.WriteLayer(repo, layer, o.Remote...); err != nil {
+		return nil, fmt.Errorf("uploading layer: %v", err)
+	}
+
+	img, err = mutate.Append(img, mutate.Addendum{
+		Layer: layer,
+		History: v1.History{
+			CreatedBy: fmt.Sprintf("%s flatten %s", use, digest),
+			Comment:   string(oldHistory),
 		},
+	})
+	if err != nil {
+		return nil, fmt.Errorf("appending layers: %v", err)
 	}
-	flattenCmd.Flags().StringVarP(&newRef, "tag", "t", "", "New tag to apply to flattened image. If not provided, push by digest to the original image repository.")
-	return flattenCmd
+
+	// Retain any annotations from the original image.
+	if len(m.Annotations) != 0 {
+		img = mutate.Annotations(img, m.Annotations).(v1.Image)
+	}
+
+	return img, nil
 }
diff --git a/pkg/crane/catalog.go b/pkg/crane/catalog.go
@@ -24,12 +24,12 @@ import (
 // Catalog returns the repositories in a registry's catalog.
 func Catalog(src string, opt ...Option) (res []string, err error) {
 	o := makeOptions(opt...)
-	reg, err := name.NewRegistry(src, o.name...)
+	reg, err := name.NewRegistry(src, o.Name...)
 	if err != nil {
 		return nil, err
 	}
 
 	// This context gets overridden by remote.WithContext, which is set by
 	// crane.WithContext.
-	return remote.Catalog(context.Background(), reg, o.remote...)
+	return remote.Catalog(context.Background(), reg, o.Remote...)
 }
diff --git a/pkg/crane/copy.go b/pkg/crane/copy.go
@@ -27,26 +27,26 @@ import (
 // Copy copies a remote image or index from src to dst.
 func Copy(src, dst string, opt ...Option) error {
 	o := makeOptions(opt...)
-	srcRef, err := name.ParseReference(src, o.name...)
+	srcRef, err := name.ParseReference(src, o.Name...)
 	if err != nil {
 		return fmt.Errorf("parsing reference %q: %v", src, err)
 	}
 
-	dstRef, err := name.ParseReference(dst, o.name...)
+	dstRef, err := name.ParseReference(dst, o.Name...)
 	if err != nil {
 		return fmt.Errorf("parsing reference for %q: %v", dst, err)
 	}
 
 	logs.Progress.Printf("Copying from %v to %v", srcRef, dstRef)
-	desc, err := remote.Get(srcRef, o.remote...)
+	desc, err := remote.Get(srcRef, o.Remote...)
 	if err != nil {
 		return fmt.Errorf("fetching %q: %v", src, err)
 	}
 
 	switch desc.MediaType {
 	case types.OCIImageIndex, types.DockerManifestList:
 		// Handle indexes separately.
-		if o.platform != nil {
+		if o.Platform != nil {
 			// If platform is explicitly set, don't copy the whole index, just the appropriate image.
 			if err := copyImage(desc, dstRef, o); err != nil {
 				return fmt.Errorf("failed to copy image: %v", err)
@@ -58,7 +58,7 @@ func Copy(src, dst string, opt ...Option) error {
 		}
 	case types.DockerManifestSchema1, types.DockerManifestSchema1Signed:
 		// Handle schema 1 images separately.
-		if err := legacy.CopySchema1(desc, srcRef, dstRef, o.remote...); err != nil {
+		if err := legacy.CopySchema1(desc, srcRef, dstRef, o.Remote...); err != nil {
 			return fmt.Errorf("failed to copy schema 1 image: %v", err)
 		}
 	default:
@@ -71,18 +71,18 @@ func Copy(src, dst string, opt ...Option) error {
 	return nil
 }
 
-func copyImage(desc *remote.Descriptor, dstRef name.Reference, o options) error {
+func copyImage(desc *remote.Descriptor, dstRef name.Reference, o Options) error {
 	img, err := desc.Image()
 	if err != nil {
 		return err
 	}
-	return remote.Write(dstRef, img, o.remote...)
+	return remote.Write(dstRef, img, o.Remote...)
 }
 
-func copyIndex(desc *remote.Descriptor, dstRef name.Reference, o options) error {
+func copyIndex(desc *remote.Descriptor, dstRef name.Reference, o Options) error {
 	idx, err := desc.ImageIndex()
 	if err != nil {
 		return err
 	}
-	return remote.WriteIndex(dstRef, idx, o.remote...)
+	return remote.WriteIndex(dstRef, idx, o.Remote...)
 }
diff --git a/pkg/crane/delete.go b/pkg/crane/delete.go
@@ -24,10 +24,10 @@ import (
 // Delete deletes the remote reference at src.
 func Delete(src string, opt ...Option) error {
 	o := makeOptions(opt...)
-	ref, err := name.ParseReference(src, o.name...)
+	ref, err := name.ParseReference(src, o.Name...)
 	if err != nil {
 		return fmt.Errorf("parsing reference %q: %v", src, err)
 	}
 
-	return remote.Delete(ref, o.remote...)
+	return remote.Delete(ref, o.Remote...)
 }