copy: set media types

When copying an image, record the compression in the BlobInfo and use
the information when updating the manifest's layer infos to set the
layers' media types correctly.

Fixes: github.com/containers/podman/issues/2013
Fixes: github.com/containers/buildah/issues/1589

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

copy/copy.go

@@ -899,6 +899,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
 		return types.BlobInfo{}, errors.Wrap(err, "Error writing blob")
 	}
 
+	// If we can modify the layer's blob, set the desired compression for it to be set in the manifest.
+	if canModifyBlob && !isConfig {
+		uploadedInfo.Compression = c.dest.DesiredLayerCompression()
+		uploadedInfo.CompressionAlgorithm = desiredCompressionFormat.Name()
+	}
+
 	// This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consumer
 	// all of the input (to compute DiffIDs), even if dest.PutBlob does not need it.
 	// So, read everything from originalLayerReader, which will cause the rest to be

image/docker_schema2.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"strings"
 
@@ -207,12 +208,17 @@ func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context) (types.Imag
 	layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors))
 	for idx := range layers {
 		layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx])
-		if m.m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType {
+		switch m.m.LayersDescriptors[idx].MediaType {
+		case manifest.DockerV2Schema2ForeignLayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable
-		} else {
-			// we assume layers are gzip'ed because docker v2s2 only deals with
-			// gzip'ed layers. However, OCI has non-gzip'ed layers as well.
+		case manifest.DockerV2SchemaLayerMediaTypeUncompressed:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayer
+		case manifest.DockerV2Schema2LayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip
+		case manifest.DockerV2Schema2LayerMediaTypeZstd:
+			layers[idx].MediaType = manifest.OCIV1MediaTypeImageLayerZstd
+		default:
+			return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType)
 		}
 	}
 

image/oci.go

@@ -3,6 +3,7 @@ package image
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 
 	"github.com/containers/image/docker/reference"
@@ -187,7 +188,18 @@ func (m *manifestOCI1) convertToManifestSchema2() (types.Image, error) {
 	layers := make([]manifest.Schema2Descriptor, len(m.m.Layers))
 	for idx := range layers {
 		layers[idx] = schema2DescriptorFromOCI1Descriptor(m.m.Layers[idx])
-		layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType
+		switch layers[idx].MediaType {
+		case imgspecv1.MediaTypeImageLayerNonDistributable:
+			layers[idx].MediaType = manifest.DockerV2Schema2ForeignLayerMediaType
+		case imgspecv1.MediaTypeImageLayer:
+			layers[idx].MediaType = manifest.DockerV2SchemaLayerMediaTypeUncompressed
+		case imgspecv1.MediaTypeImageLayerGzip:
+			layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaType
+		case manifest.OCIV1MediaTypeImageLayerZstd:
+			layers[idx].MediaType = manifest.DockerV2Schema2LayerMediaTypeZstd
+		default:
+			return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", layers[idx].MediaType)
+		}
 	}
 
 	// Rather than copying the ConfigBlob now, we just pass m.src to the

image/sourced.go

@@ -5,6 +5,7 @@ package image
 
 import (
 	"context"
+
 	"github.com/containers/image/types"
 )
 

manifest/docker_schema2.go

@@ -2,8 +2,10 @@ package manifest
 
 import (
 	"encoding/json"
+	"fmt"
 	"time"
 
+	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/pkg/strslice"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
@@ -207,7 +209,23 @@ func (m *Schema2) UpdateLayerInfos(layerInfos []types.BlobInfo) error {
 	original := m.LayersDescriptors
 	m.LayersDescriptors = make([]Schema2Descriptor, len(layerInfos))
 	for i, info := range layerInfos {
-		m.LayersDescriptors[i].MediaType = original[i].MediaType
+		switch info.Compression {
+		case types.PreserveOriginal:
+			m.LayersDescriptors[i].MediaType = original[i].MediaType
+		case types.Decompress:
+			m.LayersDescriptors[i].MediaType = DockerV2SchemaLayerMediaTypeUncompressed
+		case types.Compress:
+			switch info.CompressionAlgorithm {
+			case compression.GzipCompression:
+				m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaType
+			case compression.ZstdCompression:
+				m.LayersDescriptors[i].MediaType = DockerV2Schema2LayerMediaTypeZstd
+			default:
+				return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q fo layer %q", info.CompressionAlgorithm, info.Digest)
+			}
+		default:
+			return fmt.Errorf("Error preparing updated manifest: unknown compression instruction (%d) for layer %q", info.Compression, info.Digest)
+		}
 		m.LayersDescriptors[i].Digest = info.Digest
 		m.LayersDescriptors[i].Size = info.Size
 		m.LayersDescriptors[i].URLs = info.URLs

manifest/manifest.go

@@ -24,10 +24,17 @@ const (
 	DockerV2Schema2ConfigMediaType = "application/vnd.docker.container.image.v1+json"
 	// DockerV2Schema2LayerMediaType is the MIME type used for schema 2 layers.
 	DockerV2Schema2LayerMediaType = "application/vnd.docker.image.rootfs.diff.tar.gzip"
+	// DockerV2Schema2LayerMediaTypeZstd is the MIME type used for schema 2 layers compressed with zstd.
+	DockerV2Schema2LayerMediaTypeZstd = "application/vnd.docker.image.rootfs.diff.tar.zstd"
+	// DockerV2SchemaLayerMediaTypeUncompressed is the mediaType used for uncompressed layers.
+	DockerV2SchemaLayerMediaTypeUncompressed = "application/vnd.docker.image.rootfs.diff.tar"
 	// DockerV2ListMediaType MIME type represents Docker manifest schema 2 list
 	DockerV2ListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json"
 	// DockerV2Schema2ForeignLayerMediaType is the MIME type used for schema 2 foreign layers.
 	DockerV2Schema2ForeignLayerMediaType = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
+	// OCIV1MediaTypeImageLayerZstd is the MIME types used for OCI v1 layers compressed with zstd.
+	// Note that this MIME type is not (yet) part of the OCI v1 image specification.
+	OCIV1MediaTypeImageLayerZstd = "application/vnd.oci.image.layer.v1.tar+zstd"
 )
 
 // DefaultRequestedManifestMIMETypes is a list of MIME types a types.ImageSource

manifest/oci.go

@@ -2,7 +2,9 @@ package manifest
 
 import (
 	"encoding/json"
+	"fmt"
 
+	"github.com/containers/image/pkg/compression"
 	"github.com/containers/image/types"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go"
@@ -81,7 +83,23 @@ func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error {
 	original := m.Layers
 	m.Layers = make([]imgspecv1.Descriptor, len(layerInfos))
 	for i, info := range layerInfos {
-		m.Layers[i].MediaType = original[i].MediaType
+		switch info.Compression {
+		case types.PreserveOriginal:
+			m.Layers[i].MediaType = original[i].MediaType
+		case types.Decompress:
+			m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayer
+		case types.Compress:
+			switch info.CompressionAlgorithm {
+			case compression.GzipCompression:
+				m.Layers[i].MediaType = imgspecv1.MediaTypeImageLayerGzip
+			case compression.ZstdCompression:
+				m.Layers[i].MediaType = OCIV1MediaTypeImageLayerZstd
+			default:
+				return fmt.Errorf("Error preparing updated manifest: unknown compression algorithm %q for layer %q", info.CompressionAlgorithm, info.Digest)
+			}
+		default:
+			return fmt.Errorf("Error preparing updated manifest: unknown compression instruction (%d) for layer %q", info.Compression, info.Digest)
+		}
 		m.Layers[i].Digest = info.Digest
 		m.Layers[i].Size = info.Size
 		m.Layers[i].Annotations = info.Annotations

pkg/compression/compression.go

@@ -13,6 +13,17 @@ import (
 	"github.com/ulikunitz/xz"
 )
 
+const (
+	// GzipCompression indicates gzip compression.
+	GzipCompression = "gzip"
+	// Bzip2Compression indicates bzip2 compression.
+	Bzip2Compression = "bzip2"
+	// XzCompression indicates xz compression.
+	XzCompression = "xz"
+	// ZstdCompression indicates zstd compression.
+	ZstdCompression = "zstd"
+)
+
 // DecompressorFunc returns the decompressed stream, given a compressed stream.
 // The caller must call Close() on the decompressed stream (even if the compressed input stream does not need closing!).
 type DecompressorFunc func(io.Reader) (io.ReadCloser, error)
@@ -73,10 +84,10 @@ func (c Algorithm) Name() string {
 
 // compressionAlgos is an internal implementation detail of DetectCompression
 var compressionAlgos = []Algorithm{
-	{"gzip", []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor},             // gzip (RFC 1952)
-	{"bzip2", []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor},          // bzip2 (decompress.c:BZ2_decompress)
-	{"xz", []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor}, // xz (/usr/share/doc/xz/xz-file-format.txt)
-	{"zstd", []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor},       // zstd (http://www.zstd.net)
+	{GzipCompression, []byte{0x1F, 0x8B, 0x08}, GzipDecompressor, gzipCompressor},             // gzip (RFC 1952)
+	{Bzip2Compression, []byte{0x42, 0x5A, 0x68}, Bzip2Decompressor, bzip2Compressor},          // bzip2 (decompress.c:BZ2_decompress)
+	{XzCompression, []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, XzDecompressor, xzCompressor}, // xz (/usr/share/doc/xz/xz-file-format.txt)
+	{ZstdCompression, []byte{0x28, 0xb5, 0x2f, 0xfd}, ZstdDecompressor, zstdCompressor},       // zstd (http://www.zstd.net)
 }
 
 // AlgorithmByName returns the compressor by its name

storage/storage_image.go

@@ -345,10 +345,10 @@ func (s *storageImageDestination) Close() error {
 }
 
 func (s *storageImageDestination) DesiredLayerCompression() types.LayerCompression {
-	// We ultimately have to decompress layers to populate trees on disk,
-	// so callers shouldn't bother compressing them before handing them to
-	// us, if they're not already compressed.
-	return types.PreserveOriginal
+	// We ultimately have to decompress layers to populate trees on disk
+	// and need to explicitly ask for it here, so that the layers' MIME
+	// types can be set accordingly.
+	return types.Decompress
 }
 
 func (s *storageImageDestination) computeNextBlobCacheFile() string {

types/types.go

@@ -8,7 +8,7 @@ import (
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/pkg/compression"
 	"github.com/opencontainers/go-digest"
-	"github.com/opencontainers/image-spec/specs-go/v1"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ImageTransport is a top-level namespace for ways to to store/load an image.
@@ -91,14 +91,29 @@ type ImageReference interface {
 	DeleteImage(ctx context.Context, sys *SystemContext) error
 }
 
+// LayerCompression indicates if layers must be compressed, decompressed or preserved
+type LayerCompression int
+
+const (
+	// PreserveOriginal indicates the layer must be preserved, ie
+	// no compression or decompression.
+	PreserveOriginal LayerCompression = iota
+	// Decompress indicates the layer must be decompressed
+	Decompress
+	// Compress indicates the layer must be compressed
+	Compress
+)
+
 // BlobInfo collects known information about a blob (layer/config).
 // In some situations, some fields may be unknown, in others they may be mandatory; documenting an “unknown” value here does not override that.
 type BlobInfo struct {
-	Digest      digest.Digest // "" if unknown.
-	Size        int64         // -1 if unknown
-	URLs        []string
-	Annotations map[string]string
-	MediaType   string
+	Digest               digest.Digest // "" if unknown.
+	Size                 int64         // -1 if unknown
+	URLs                 []string
+	Annotations          map[string]string
+	MediaType            string
+	Compression          LayerCompression
+	CompressionAlgorithm string // as defined pkg/compression.{GzipCompression, ZstdCompression, ...}
 }
 
 // BICTransportScope encapsulates transport-dependent representation of a “scope” where blobs are or are not present.
@@ -212,19 +227,6 @@ type ImageSource interface {
 	LayerInfosForCopy(ctx context.Context) ([]BlobInfo, error)
 }
 
-// LayerCompression indicates if layers must be compressed, decompressed or preserved
-type LayerCompression int
-
-const (
-	// PreserveOriginal indicates the layer must be preserved, ie
-	// no compression or decompression.
-	PreserveOriginal LayerCompression = iota
-	// Decompress indicates the layer must be decompressed
-	Decompress
-	// Compress indicates the layer must be compressed
-	Compress
-)
-
 // ImageDestination is a service, possibly remote (= slow), to store components of a single image.
 //
 // There is a specific required order for some of the calls: