Support a mode to emit estargz automagically. (#871)

* Have tarball.LayerFromOpener support estargz

* Add docs for env var, use upstream constant

* Fix comment on WithEstargzOptions

README.md

@@ -26,6 +26,15 @@ There are a number of packages for reading/writing these interfaces from/to vari
 The main focus has been registry interactions (hence the name) via the [`remote`](pkg/v1/remote) package,
 but we have implemented other formats as we needed them to interoperate with various tools.
 
+### Experiments
+
+Over time, we will add new functionality under experimental environment variables listed here.
+
+| Env Var | Value(s) | What is does |
+|---------|----------|--------------|
+| `GGCR_EXPERIMENT_ESTARGZ` | `"1"` | When enabled this experiment will direct `tarball.LayerFromOpener` to emit [estargz](https://github.com/opencontainers/image-spec/issues/815) compatible layers, which enable them to be lazily loaded by an appropriately configured containerd. |
+
+
 ### `v1.Image`
 
 #### Sources

pkg/v1/internal/estargz/estargz.go

@@ -23,6 +23,9 @@ import (
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 )
 
+// Assert that what we're returning is an io.ReadCloser
+var _ io.ReadCloser = (*estargz.Blob)(nil)
+
 // ReadCloser reads uncompressed tarball input from the io.ReadCloser and
 // returns:
 //  * An io.ReadCloser from which compressed data may be read, and
@@ -31,7 +34,7 @@ import (
 //
 // Refer to estargz for the options:
 // https://pkg.go.dev/github.com/containerd/stargz-snapshotter@v0.2.0/estargz#Option
-func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (io.ReadCloser, v1.Hash, error) {
+func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) {
 	defer r.Close()
 
 	// TODO(#876): Avoid buffering into memory.

pkg/v1/tarball/layer.go

@@ -22,9 +22,13 @@ import (
 	"os"
 	"sync"
 
+	"github.com/containerd/stargz-snapshotter/estargz"
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/internal/and"
+	gestargz "github.com/google/go-containerregistry/pkg/v1/internal/estargz"
 	ggzip "github.com/google/go-containerregistry/pkg/v1/internal/gzip"
 	"github.com/google/go-containerregistry/pkg/v1/types"
+	"github.com/google/go-containerregistry/pkg/v1/v1util"
 )
 
 type layer struct {
@@ -34,28 +38,50 @@ type layer struct {
 	compressedopener   Opener
 	uncompressedopener Opener
 	compression        int
+	annotations        map[string]string
+	estgzopts          []estargz.Option
 }
 
+// Descriptor implements partial.withDescriptor.
+func (l *layer) Descriptor() (*v1.Descriptor, error) {
+	digest, err := l.Digest()
+	if err != nil {
+		return nil, err
+	}
+	return &v1.Descriptor{
+		Size:        l.size,
+		Digest:      digest,
+		Annotations: l.annotations,
+		MediaType:   types.DockerLayer,
+	}, nil
+}
+
+// Digest implements v1.Layer
 func (l *layer) Digest() (v1.Hash, error) {
 	return l.digest, nil
 }
 
+// DiffID implements v1.Layer
 func (l *layer) DiffID() (v1.Hash, error) {
 	return l.diffID, nil
 }
 
+// Compressed implements v1.Layer
 func (l *layer) Compressed() (io.ReadCloser, error) {
 	return l.compressedopener()
 }
 
+// Uncompressed implements v1.Layer
 func (l *layer) Uncompressed() (io.ReadCloser, error) {
 	return l.uncompressedopener()
 }
 
+// Size implements v1.Layer
 func (l *layer) Size() (int64, error) {
 	return l.size, nil
 }
 
+// MediaType implements v1.Layer
 func (l *layer) MediaType() (types.MediaType, error) {
 	return types.DockerLayer, nil
 }
@@ -98,6 +124,15 @@ func WithCompressedCaching(l *layer) {
 	}
 }
 
+// WithEstargzOptions is a functional option that allow the caller to pass
+// through estargz.Options to the underlying compression layer.  This is
+// only meaningful when estargz is enabled.
+func WithEstargzOptions(opts ...estargz.Option) LayerOption {
+	return func(l *layer) {
+		l.estgzopts = opts
+	}
+}
+
 // LayerFromFile returns a v1.Layer given a tarball
 func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) {
 	opener := func() (io.ReadCloser, error) {
@@ -130,6 +165,7 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 
 	layer := &layer{
 		compression: gzip.BestSpeed,
+		annotations: make(map[string]string, 1),
 	}
 
 	if compressed {
@@ -141,6 +177,38 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 			}
 			return ggzip.UnzipReadCloser(urc)
 		}
+	} else if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" {
+		layer.compressedopener = func() (io.ReadCloser, error) {
+			crc, err := opener()
+			if err != nil {
+				return nil, err
+			}
+			eopts := append(layer.estgzopts, estargz.WithCompressionLevel(layer.compression))
+			rc, h, err := gestargz.ReadCloser(crc, eopts...)
+			if err != nil {
+				return nil, err
+			}
+			layer.annotations[estargz.TOCJSONDigestAnnotation] = h.String()
+			return &and.ReadCloser{
+				Reader: rc,
+				CloseFunc: func() error {
+					err := rc.Close()
+					if err != nil {
+						return err
+					}
+					// As an optimization, leverage the DiffID exposed by the estargz ReadCloser
+					layer.diffID, err = v1.NewHash(rc.DiffID().String())
+					return err
+				},
+			}, nil
+		}
+		layer.uncompressedopener = func() (io.ReadCloser, error) {
+			urc, err := layer.compressedopener()
+			if err != nil {
+				return nil, err
+			}
+			return v1util.GunzipReadCloser(urc)
+		}
 	} else {
 		layer.uncompressedopener = opener
 		layer.compressedopener = func() (io.ReadCloser, error) {
@@ -160,8 +228,11 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
 		return nil, err
 	}
 
-	if layer.diffID, err = computeDiffID(layer.uncompressedopener); err != nil {
-		return nil, err
+	empty := v1.Hash{}
+	if layer.diffID == empty {
+		if layer.diffID, err = computeDiffID(layer.uncompressedopener); err != nil {
+			return nil, err
+		}
 	}
 
 	return layer, nil

pkg/v1/tarball/layer_test.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"testing"
 
+	"github.com/containerd/stargz-snapshotter/estargz"
 	"github.com/google/go-containerregistry/pkg/internal/compare"
 	"github.com/google/go-containerregistry/pkg/v1/validate"
 )
@@ -77,6 +78,96 @@ func TestLayerFromFile(t *testing.T) {
 	}
 }
 
+func TestLayerFromFileEstargz(t *testing.T) {
+	os.Setenv("GGCR_EXPERIMENT_ESTARGZ", "1")
+	defer os.Unsetenv("GGCR_EXPERIMENT_ESTARGZ")
+	setupFixtures(t)
+	defer teardownFixtures(t)
+
+	tarLayer, err := LayerFromFile("testdata/content.tar")
+	if err != nil {
+		t.Fatalf("Unable to create layer from tar file: %v", err)
+	}
+
+	if err := validate.Layer(tarLayer); err != nil {
+		t.Errorf("validate.Layer(tarLayer): %v", err)
+	}
+
+	tarLayerDefaultCompression, err := LayerFromFile("testdata/content.tar", WithCompressionLevel(gzip.DefaultCompression))
+	if err != nil {
+		t.Fatalf("Unable to create layer with 'Default' compression from tar file: %v", err)
+	}
+	descriptorDefaultCompression, err := tarLayerDefaultCompression.(*layer).Descriptor()
+	if err != nil {
+		t.Fatalf("Descriptor() = %v", err)
+	} else if len(descriptorDefaultCompression.Annotations) != 1 {
+		t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorDefaultCompression.Annotations)
+	}
+
+	defaultDigest, err := tarLayerDefaultCompression.Digest()
+	if err != nil {
+		t.Fatal("Unable to generate digest with 'Default' compression", err)
+	}
+
+	tarLayerSpeedCompression, err := LayerFromFile("testdata/content.tar", WithCompressionLevel(gzip.BestSpeed))
+	if err != nil {
+		t.Fatalf("Unable to create layer with 'BestSpeed' compression from tar file: %v", err)
+	}
+	descriptorSpeedCompression, err := tarLayerSpeedCompression.(*layer).Descriptor()
+	if err != nil {
+		t.Fatalf("Descriptor() = %v", err)
+	} else if len(descriptorSpeedCompression.Annotations) != 1 {
+		t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorSpeedCompression.Annotations)
+	}
+
+	speedDigest, err := tarLayerSpeedCompression.Digest()
+	if err != nil {
+		t.Fatal("Unable to generate digest with 'BestSpeed' compression", err)
+	}
+
+	if defaultDigest.String() == speedDigest.String() {
+		t.Errorf("expected digests to differ: %s", defaultDigest.String())
+	}
+
+	if descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation] == descriptorSpeedCompression.Annotations[estargz.TOCJSONDigestAnnotation] {
+		t.Errorf("wanted different toc digests got default: %s, speed: %s",
+			descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation],
+			descriptorSpeedCompression.Annotations[estargz.TOCJSONDigestAnnotation])
+	}
+
+	tarLayerPrioritizedFiles, err := LayerFromFile("testdata/content.tar",
+		// We compare with default, so pass for apples-to-apples comparison.
+		WithCompressionLevel(gzip.DefaultCompression),
+		// By passing a list of priority files, we expect the layer to be different.
+		WithEstargzOptions(estargz.WithPrioritizedFiles([]string{
+			"./bat",
+		})))
+	if err != nil {
+		t.Fatalf("Unable to create layer with prioritized files from tar file: %v", err)
+	}
+	descriptorPrioritizedFiles, err := tarLayerPrioritizedFiles.(*layer).Descriptor()
+	if err != nil {
+		t.Fatalf("Descriptor() = %v", err)
+	} else if len(descriptorPrioritizedFiles.Annotations) != 1 {
+		t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorPrioritizedFiles.Annotations)
+	}
+
+	prioritizedDigest, err := tarLayerPrioritizedFiles.Digest()
+	if err != nil {
+		t.Fatal("Unable to generate digest with prioritized files", err)
+	}
+
+	if defaultDigest.String() == prioritizedDigest.String() {
+		t.Errorf("expected digests to differ: %s", defaultDigest.String())
+	}
+
+	if descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation] == descriptorPrioritizedFiles.Annotations[estargz.TOCJSONDigestAnnotation] {
+		t.Errorf("wanted different toc digests got default: %s, prioritized: %s",
+			descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation],
+			descriptorPrioritizedFiles.Annotations[estargz.TOCJSONDigestAnnotation])
+	}
+}
+
 func TestLayerFromOpenerReader(t *testing.T) {
 	setupFixtures(t)
 	defer teardownFixtures(t)