layer.go

// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tarball

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"sync"

	"github.com/containerd/stargz-snapshotter/estargz"
	"github.com/google/go-containerregistry/internal/and"
	gestargz "github.com/google/go-containerregistry/internal/estargz"
	ggzip "github.com/google/go-containerregistry/internal/gzip"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/types"
)

type layer struct {
	digest             v1.Hash
	diffID             v1.Hash
	size               int64
	compressedopener   Opener
	uncompressedopener Opener
	compression        int
	annotations        map[string]string
	estgzopts          []estargz.Option
	mediaType          types.MediaType
}

// Descriptor implements partial.withDescriptor.
func (l *layer) Descriptor() (*v1.Descriptor, error) {
	digest, err := l.Digest()
	if err != nil {
		return nil, err
	}
	return &v1.Descriptor{
		Size:        l.size,
		Digest:      digest,
		Annotations: l.annotations,
		MediaType:   l.mediaType,
	}, nil
}

// Digest implements v1.Layer
func (l *layer) Digest() (v1.Hash, error) {
	return l.digest, nil
}

// DiffID implements v1.Layer
func (l *layer) DiffID() (v1.Hash, error) {
	return l.diffID, nil
}

// Compressed implements v1.Layer
func (l *layer) Compressed() (io.ReadCloser, error) {
	return l.compressedopener()
}

// Uncompressed implements v1.Layer
func (l *layer) Uncompressed() (io.ReadCloser, error) {
	return l.uncompressedopener()
}

// Size implements v1.Layer
func (l *layer) Size() (int64, error) {
	return l.size, nil
}

// MediaType implements v1.Layer
func (l *layer) MediaType() (types.MediaType, error) {
	return l.mediaType, nil
}

// LayerOption applies options to layer
type LayerOption func(*layer)

// WithCompressionLevel is a functional option for overriding the default
// compression level used for compressing uncompressed tarballs.
func WithCompressionLevel(level int) LayerOption {
	return func(l *layer) {
		l.compression = level
	}
}

// WithMediaType is a functional option for overriding the layer's media type.
func WithMediaType(mt types.MediaType) LayerOption {
	return func(l *layer) {
		l.mediaType = mt
	}
}

// WithCompressedCaching is a functional option that overrides the
// logic for accessing the compressed bytes to memoize the result
// and avoid expensive repeated gzips.
func WithCompressedCaching(l *layer) {
	var once sync.Once
	var err error

	buf := bytes.NewBuffer(nil)
	og := l.compressedopener

	l.compressedopener = func() (io.ReadCloser, error) {
		once.Do(func() {
			var rc io.ReadCloser
			rc, err = og()
			if err == nil {
				defer rc.Close()
				_, err = io.Copy(buf, rc)
			}
		})
		if err != nil {
			return nil, err
		}

		return ioutil.NopCloser(bytes.NewBuffer(buf.Bytes())), nil
	}
}

// WithEstargzOptions is a functional option that allow the caller to pass
// through estargz.Options to the underlying compression layer.  This is
// only meaningful when estargz is enabled.
func WithEstargzOptions(opts ...estargz.Option) LayerOption {
	return func(l *layer) {
		l.estgzopts = opts
	}
}

// WithEstargz is a functional option that explicitly enables estargz support.
func WithEstargz(l *layer) {
	oguncompressed := l.uncompressedopener
	estargz := func() (io.ReadCloser, error) {
		crc, err := oguncompressed()
		if err != nil {
			return nil, err
		}
		eopts := append(l.estgzopts, estargz.WithCompressionLevel(l.compression))
		rc, h, err := gestargz.ReadCloser(crc, eopts...)
		if err != nil {
			return nil, err
		}
		l.annotations[estargz.TOCJSONDigestAnnotation] = h.String()
		return &and.ReadCloser{
			Reader: rc,
			CloseFunc: func() error {
				err := rc.Close()
				if err != nil {
					return err
				}
				// As an optimization, leverage the DiffID exposed by the estargz ReadCloser
				l.diffID, err = v1.NewHash(rc.DiffID().String())
				return err
			},
		}, nil
	}
	uncompressed := func() (io.ReadCloser, error) {
		urc, err := estargz()
		if err != nil {
			return nil, err
		}
		return ggzip.UnzipReadCloser(urc)
	}

	l.compressedopener = estargz
	l.uncompressedopener = uncompressed
}

// LayerFromFile returns a v1.Layer given a tarball
func LayerFromFile(path string, opts ...LayerOption) (v1.Layer, error) {
	opener := func() (io.ReadCloser, error) {
		return os.Open(path)
	}
	return LayerFromOpener(opener, opts...)
}

// LayerFromOpener returns a v1.Layer given an Opener function.
// The Opener may return either an uncompressed tarball (common),
// or a compressed tarball (uncommon).
//
// When using this in conjunction with something like remote.Write
// the uncompressed path may end up gzipping things multiple times:
//  1. Compute the layer SHA256
//  2. Upload the compressed layer.
// Since gzip can be expensive, we support an option to memoize the
// compression that can be passed here: tarball.WithCompressedCaching
func LayerFromOpener(opener Opener, opts ...LayerOption) (v1.Layer, error) {
	rc, err := opener()
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	compressed, err := ggzip.Is(rc)
	if err != nil {
		return nil, err
	}

	layer := &layer{
		compression: gzip.BestSpeed,
		annotations: make(map[string]string, 1),
		mediaType:   types.DockerLayer,
	}

	if estgz := os.Getenv("GGCR_EXPERIMENT_ESTARGZ"); estgz == "1" {
		opts = append([]LayerOption{WithEstargz}, opts...)
	}

	if compressed {
		layer.compressedopener = opener
		layer.uncompressedopener = func() (io.ReadCloser, error) {
			urc, err := opener()
			if err != nil {
				return nil, err
			}
			return ggzip.UnzipReadCloser(urc)
		}
	} else {
		layer.uncompressedopener = opener
		layer.compressedopener = func() (io.ReadCloser, error) {
			crc, err := opener()
			if err != nil {
				return nil, err
			}
			return ggzip.ReadCloserLevel(crc, layer.compression), nil
		}
	}

	for _, opt := range opts {
		opt(layer)
	}

	if layer.digest, layer.size, err = computeDigest(layer.compressedopener); err != nil {
		return nil, err
	}

	empty := v1.Hash{}
	if layer.diffID == empty {
		if layer.diffID, err = computeDiffID(layer.uncompressedopener); err != nil {
			return nil, err
		}
	}

	return layer, nil
}

// LayerFromReader returns a v1.Layer given a io.Reader.
//
// The reader's contents are read and buffered to a temp file in the process.
//
// Deprecated: Use LayerFromOpener or stream.NewLayer instead, if possible.
func LayerFromReader(reader io.Reader, opts ...LayerOption) (v1.Layer, error) {
	tmp, err := ioutil.TempFile("", "")
	if err != nil {
		return nil, fmt.Errorf("creating temp file to buffer reader: %w", err)
	}
	if _, err := io.Copy(tmp, reader); err != nil {
		return nil, fmt.Errorf("writing temp file to buffer reader: %w", err)
	}
	return LayerFromFile(tmp.Name(), opts...)
}

func computeDigest(opener Opener) (v1.Hash, int64, error) {
	rc, err := opener()
	if err != nil {
		return v1.Hash{}, 0, err
	}
	defer rc.Close()

	return v1.SHA256(rc)
}

func computeDiffID(opener Opener) (v1.Hash, error) {
	rc, err := opener()
	if err != nil {
		return v1.Hash{}, err
	}
	defer rc.Close()

	digest, _, err := v1.SHA256(rc)
	return digest, err
}