From 687e0e365894acc9096d2f581c58884ef258bc03 Mon Sep 17 00:00:00 2001 From: Matt Moore Date: Thu, 17 Dec 2020 13:18:53 -0800 Subject: [PATCH] Support a mode to emit estargz automagically. (#871) * Have tarball.LayerFromOpener support estargz * Add docs for env var, use upstream constant * Fix comment on WithEstargzOptions --- README.md | 9 +++ pkg/v1/internal/estargz/estargz.go | 5 +- pkg/v1/tarball/layer.go | 75 +++++++++++++++++++++++- pkg/v1/tarball/layer_test.go | 91 ++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1b5442395..f56a64cba 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,15 @@ There are a number of packages for reading/writing these interfaces from/to vari The main focus has been registry interactions (hence the name) via the [`remote`](pkg/v1/remote) package, but we have implemented other formats as we needed them to interoperate with various tools. +### Experiments + +Over time, we will add new functionality under experimental environment variables listed here. + +| Env Var | Value(s) | What is does | +|---------|----------|--------------| +| `GGCR_EXPERIMENT_ESTARGZ` | `"1"` | When enabled this experiment will direct `tarball.LayerFromOpener` to emit [estargz](https://github.com/opencontainers/image-spec/issues/815) compatible layers, which enable them to be lazily loaded by an appropriately configured containerd. | + + ### `v1.Image` #### Sources diff --git a/pkg/v1/internal/estargz/estargz.go b/pkg/v1/internal/estargz/estargz.go index 84e436fb6..6404d3d95 100644 --- a/pkg/v1/internal/estargz/estargz.go +++ b/pkg/v1/internal/estargz/estargz.go @@ -23,6 +23,9 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" ) +// Assert that what we're returning is an io.ReadCloser +var _ io.ReadCloser = (*estargz.Blob)(nil) + // ReadCloser reads uncompressed tarball input from the io.ReadCloser and // returns: // * An io.ReadCloser from which compressed data may be read, and @@ -31,7 +34,7 @@ import ( // // Refer to estargz for the options: // https://pkg.go.dev/github.com/containerd/stargz-snapshotter@v0.2.0/estargz#Option -func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (io.ReadCloser, v1.Hash, error) { +func ReadCloser(r io.ReadCloser, opts ...estargz.Option) (*estargz.Blob, v1.Hash, error) { defer r.Close() // TODO(#876): Avoid buffering into memory. diff --git a/pkg/v1/tarball/layer.go b/pkg/v1/tarball/layer.go index a06be349a..e6be337a5 100644 --- a/pkg/v1/tarball/layer.go +++ b/pkg/v1/tarball/layer.go @@ -22,9 +22,13 @@ import ( "os" "sync" + "github.com/containerd/stargz-snapshotter/estargz" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/internal/and" + gestargz "github.com/google/go-containerregistry/pkg/v1/internal/estargz" ggzip "github.com/google/go-containerregistry/pkg/v1/internal/gzip" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/go-containerregistry/pkg/v1/v1util" ) type layer struct { @@ -34,28 +38,50 @@ type layer struct { compressedopener Opener uncompressedopener Opener compression int + annotations map[string]string + estgzopts []estargz.Option } +// Descriptor implements partial.withDescriptor. +func (l *layer) Descriptor() (*v1.Descriptor, error) { + digest, err := l.Digest() + if err != nil { + return nil, err + } + return &v1.Descriptor{ + Size: l.size, + Digest: digest, + Annotations: l.annotations, + MediaType: types.DockerLayer, + }, nil +} + +// Digest implements v1.Layer func (l *layer) Digest() (v1.Hash, error) { return l.digest, nil } +// DiffID implements v1.Layer func (l *layer) DiffID() (v1.Hash, error) { return l.diffID, nil } +// Compressed implements v1.Layer func (l *layer) Compressed() (io.ReadCloser, error) { return l.compressedopener() } +// Uncompressed implements v1.Layer func (l *layer) Uncompressed() (io.ReadCloser, error) { return l.uncompressedopener() } +// Size implements v1.Layer func (l *layer) Size() (int64, error) { return l.size, nil } +// MediaType implements v1.Layer func (l *layer) MediaType() (types.MediaType, error) { return types.DockerLayer, nil } @@ -98,6 +124,15 @@ func WithCompressedCaching(l *layer) { } } +// WithEstargzOptions is a functional option that allow the caller to pass +// through estargz.Options to the underlying compression layer. This is +// only meaningful when estargz is enabled. +func WithEstargzOptions(opts ...estargz.Option) LayerOption { + return func(l *layer) { + l.estgzopts = opts + } +} + // LayerFromFile returns a v1.Layer given a tarball func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) { opener := func() (io.ReadCloser, error) { @@ -130,6 +165,7 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { layer := &layer{ compression: gzip.BestSpeed, + annotations: make(map[string]string, 1), } if compressed { @@ -141,6 +177,38 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { } return ggzip.UnzipReadCloser(urc) } + } else if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" { + layer.compressedopener = func() (io.ReadCloser, error) { + crc, err := opener() + if err != nil { + return nil, err + } + eopts := append(layer.estgzopts, estargz.WithCompressionLevel(layer.compression)) + rc, h, err := gestargz.ReadCloser(crc, eopts...) + if err != nil { + return nil, err + } + layer.annotations[estargz.TOCJSONDigestAnnotation] = h.String() + return &and.ReadCloser{ + Reader: rc, + CloseFunc: func() error { + err := rc.Close() + if err != nil { + return err + } + // As an optimization, leverage the DiffID exposed by the estargz ReadCloser + layer.diffID, err = v1.NewHash(rc.DiffID().String()) + return err + }, + }, nil + } + layer.uncompressedopener = func() (io.ReadCloser, error) { + urc, err := layer.compressedopener() + if err != nil { + return nil, err + } + return v1util.GunzipReadCloser(urc) + } } else { layer.uncompressedopener = opener layer.compressedopener = func() (io.ReadCloser, error) { @@ -160,8 +228,11 @@ func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) { return nil, err } - if layer.diffID, err = computeDiffID(layer.uncompressedopener); err != nil { - return nil, err + empty := v1.Hash{} + if layer.diffID == empty { + if layer.diffID, err = computeDiffID(layer.uncompressedopener); err != nil { + return nil, err + } } return layer, nil diff --git a/pkg/v1/tarball/layer_test.go b/pkg/v1/tarball/layer_test.go index aaf3d679b..114eb30d1 100644 --- a/pkg/v1/tarball/layer_test.go +++ b/pkg/v1/tarball/layer_test.go @@ -22,6 +22,7 @@ import ( "os" "testing" + "github.com/containerd/stargz-snapshotter/estargz" "github.com/google/go-containerregistry/pkg/internal/compare" "github.com/google/go-containerregistry/pkg/v1/validate" ) @@ -77,6 +78,96 @@ func TestLayerFromFile(t *testing.T) { } } +func TestLayerFromFileEstargz(t *testing.T) { + os.Setenv("GGCR_EXPERIMENT_ESTARGZ", "1") + defer os.Unsetenv("GGCR_EXPERIMENT_ESTARGZ") + setupFixtures(t) + defer teardownFixtures(t) + + tarLayer, err := LayerFromFile("testdata/content.tar") + if err != nil { + t.Fatalf("Unable to create layer from tar file: %v", err) + } + + if err := validate.Layer(tarLayer); err != nil { + t.Errorf("validate.Layer(tarLayer): %v", err) + } + + tarLayerDefaultCompression, err := LayerFromFile("testdata/content.tar", WithCompressionLevel(gzip.DefaultCompression)) + if err != nil { + t.Fatalf("Unable to create layer with 'Default' compression from tar file: %v", err) + } + descriptorDefaultCompression, err := tarLayerDefaultCompression.(*layer).Descriptor() + if err != nil { + t.Fatalf("Descriptor() = %v", err) + } else if len(descriptorDefaultCompression.Annotations) != 1 { + t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorDefaultCompression.Annotations) + } + + defaultDigest, err := tarLayerDefaultCompression.Digest() + if err != nil { + t.Fatal("Unable to generate digest with 'Default' compression", err) + } + + tarLayerSpeedCompression, err := LayerFromFile("testdata/content.tar", WithCompressionLevel(gzip.BestSpeed)) + if err != nil { + t.Fatalf("Unable to create layer with 'BestSpeed' compression from tar file: %v", err) + } + descriptorSpeedCompression, err := tarLayerSpeedCompression.(*layer).Descriptor() + if err != nil { + t.Fatalf("Descriptor() = %v", err) + } else if len(descriptorSpeedCompression.Annotations) != 1 { + t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorSpeedCompression.Annotations) + } + + speedDigest, err := tarLayerSpeedCompression.Digest() + if err != nil { + t.Fatal("Unable to generate digest with 'BestSpeed' compression", err) + } + + if defaultDigest.String() == speedDigest.String() { + t.Errorf("expected digests to differ: %s", defaultDigest.String()) + } + + if descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation] == descriptorSpeedCompression.Annotations[estargz.TOCJSONDigestAnnotation] { + t.Errorf("wanted different toc digests got default: %s, speed: %s", + descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation], + descriptorSpeedCompression.Annotations[estargz.TOCJSONDigestAnnotation]) + } + + tarLayerPrioritizedFiles, err := LayerFromFile("testdata/content.tar", + // We compare with default, so pass for apples-to-apples comparison. + WithCompressionLevel(gzip.DefaultCompression), + // By passing a list of priority files, we expect the layer to be different. + WithEstargzOptions(estargz.WithPrioritizedFiles([]string{ + "./bat", + }))) + if err != nil { + t.Fatalf("Unable to create layer with prioritized files from tar file: %v", err) + } + descriptorPrioritizedFiles, err := tarLayerPrioritizedFiles.(*layer).Descriptor() + if err != nil { + t.Fatalf("Descriptor() = %v", err) + } else if len(descriptorPrioritizedFiles.Annotations) != 1 { + t.Errorf("Annotations = %#v, wanted 1 annotation", descriptorPrioritizedFiles.Annotations) + } + + prioritizedDigest, err := tarLayerPrioritizedFiles.Digest() + if err != nil { + t.Fatal("Unable to generate digest with prioritized files", err) + } + + if defaultDigest.String() == prioritizedDigest.String() { + t.Errorf("expected digests to differ: %s", defaultDigest.String()) + } + + if descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation] == descriptorPrioritizedFiles.Annotations[estargz.TOCJSONDigestAnnotation] { + t.Errorf("wanted different toc digests got default: %s, prioritized: %s", + descriptorDefaultCompression.Annotations[estargz.TOCJSONDigestAnnotation], + descriptorPrioritizedFiles.Annotations[estargz.TOCJSONDigestAnnotation]) + } +} + func TestLayerFromOpenerReader(t *testing.T) { setupFixtures(t) defer teardownFixtures(t)