media type checks

When copying an image, record the compression in the BlobInfo and use
the information when updating the manifest's layer infos to set the
layers' media types correctly.

Also check for supported media types when parsing a v2s2/OCI1 manifest.

Note that consumers of the containers/image library need to update
opencontainers/image-spec to commit 775207bd45b6cb8153ce218cc59351799217451f.

Fixes: github.com/containers/podman/issues/2013
Fixes: github.com/containers/buildah/issues/1589

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

copy/copy.go

@@ -911,6 +911,12 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
 		return types.BlobInfo{}, errors.Wrap(err, "Error writing blob")
 	}
 
+	uploadedInfo.CompressionOperation = compressionOperation
+	// If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest.
+	if canModifyBlob && !isConfig {
+		uploadedInfo.CompressionAlgorithm = &desiredCompressionFormat
+	}
+
 	// This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consumer
 	// all of the input (to compute DiffIDs), even if dest.PutBlob does not need it.
 	// So, read everything from originalLayerReader, which will cause the rest to be

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/mattn/go-isatty v0.0.4 // indirect
 	github.com/mtrmac/gpgme v0.0.0-20170102180018-b2432428689c
 	github.com/opencontainers/go-digest v1.0.0-rc1
-	github.com/opencontainers/image-spec v1.0.0
+	github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6
 	github.com/opencontainers/selinux v1.2.2
 	github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913
 	github.com/pkg/errors v0.8.1

go.sum

@@ -75,6 +75,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/image-spec v1.0.0 h1:jcw3cCH887bLKETGYpv8afogdYchbShR0eH6oD9d5PQ=
 github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
+github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 h1:yN8BPXVwMBAm3Cuvh1L5XE8XpvYRMdsVLd82ILprhUU=
+github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v1.0.0-rc8 h1:dDCFes8Hj1r/i5qnypONo5jdOme/8HWZC/aNDyhECt0=
 github.com/opencontainers/runc v1.0.0-rc8/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/selinux v1.2.2 h1:Kx9J6eDG5/24A6DtUquGSpJQ+m2MUTahn4FtGEe8bFg=
@@ -98,6 +101,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -142,6 +146,7 @@ gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+p
 gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.0.0-20170208141851-a3f3340b5840 h1:BftvRMCaj0KX6UeD7gnNJv0W8b4HAYTEWes978CoWlY=
 gopkg.in/yaml.v2 v2.0.0-20170208141851-a3f3340b5840/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gotest.tools v0.0.0-20190624233834-05ebafbffc79 h1:C+K4iPg1rIvmCf4JjelkbWv2jeWevEwp05Lz8XfTYgE=
 gotest.tools v0.0.0-20190624233834-05ebafbffc79/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90=

image/docker_schema2.go

@@ -6,6 +6,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"strings"
 
@@ -207,12 +208,17 @@ func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context) (types.Imag
 	layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors))
 	for idx := range layers {
 		layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx])
-		if m.m.LayersDescriptors[idx].MediaType == manifest.DockerV2Schema2ForeignLayerMediaType {
+		switch m.m.LayersDescriptors[idx].MediaType {
+		case manifest.DockerV2Schema2ForeignLayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable
-		} else {
-			// we assume layers are gzip'ed because docker v2s2 only deals with
-			// gzip'ed layers. However, OCI has non-gzip'ed layers as well.
+		case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableGzip
+		case manifest.DockerV2SchemaLayerMediaTypeUncompressed:
+			layers[idx].MediaType = imgspecv1.MediaTypeImageLayer
+		case manifest.DockerV2Schema2LayerMediaType:
 			layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip
+		default:
+			return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType)
 		}
 	}
 

image/docker_schema2_test.go

@@ -45,12 +45,16 @@ func (f unusedImageSource) LayerInfosForCopy(ctx context.Context) ([]types.BlobI
 	panic("Unexpected call to a mock function")
 }
 
-func manifestSchema2FromFixture(t *testing.T, src types.ImageSource, fixture string) genericManifest {
+func manifestSchema2FromFixture(t *testing.T, src types.ImageSource, fixture string, mustFail bool) genericManifest {
 	manifest, err := ioutil.ReadFile(filepath.Join("fixtures", fixture))
 	require.NoError(t, err)
 
 	m, err := manifestSchema2FromManifest(src, manifest)
-	require.NoError(t, err)
+	if mustFail {
+		require.Error(t, err)
+	} else {
+		require.NoError(t, err)
+	}
 	return m
 }
 
@@ -91,7 +95,7 @@ func manifestSchema2FromComponentsLikeFixture(configBlob []byte) genericManifest
 func TestManifestSchema2FromManifest(t *testing.T) {
 	// This just tests that the JSON can be loaded; we test that the parsed
 	// values are correctly returned in tests for the individual getter methods.
-	_ = manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json")
+	_ = manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json", false)
 
 	_, err := manifestSchema2FromManifest(nil, []byte{})
 	assert.Error(t, err)
@@ -105,7 +109,7 @@ func TestManifestSchema2FromComponents(t *testing.T) {
 
 func TestManifestSchema2Serialize(t *testing.T) {
 	for _, m := range []genericManifest{
-		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json"),
+		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json", false),
 		manifestSchema2FromComponentsLikeFixture(nil),
 	} {
 		serialized, err := m.serialize()
@@ -129,7 +133,7 @@ func TestManifestSchema2Serialize(t *testing.T) {
 
 func TestManifestSchema2ManifestMIMEType(t *testing.T) {
 	for _, m := range []genericManifest{
-		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json"),
+		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json", false),
 		manifestSchema2FromComponentsLikeFixture(nil),
 	} {
 		assert.Equal(t, manifest.DockerV2Schema2MediaType, m.manifestMIMEType())
@@ -138,7 +142,7 @@ func TestManifestSchema2ManifestMIMEType(t *testing.T) {
 
 func TestManifestSchema2ConfigInfo(t *testing.T) {
 	for _, m := range []genericManifest{
-		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json"),
+		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json", false),
 		manifestSchema2FromComponentsLikeFixture(nil),
 	} {
 		assert.Equal(t, types.BlobInfo{
@@ -195,7 +199,7 @@ func TestManifestSchema2ConfigBlob(t *testing.T) {
 		} else {
 			src = nil
 		}
-		m := manifestSchema2FromFixture(t, src, "schema2.json")
+		m := manifestSchema2FromFixture(t, src, "schema2.json", false)
 		blob, err := m.ConfigBlob(context.Background())
 		if c.blob != nil {
 			assert.NoError(t, err)
@@ -219,7 +223,7 @@ func TestManifestSchema2ConfigBlob(t *testing.T) {
 
 func TestManifestSchema2LayerInfo(t *testing.T) {
 	for _, m := range []genericManifest{
-		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json"),
+		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json", false),
 		manifestSchema2FromComponentsLikeFixture(nil),
 	} {
 		assert.Equal(t, []types.BlobInfo{
@@ -254,7 +258,7 @@ func TestManifestSchema2LayerInfo(t *testing.T) {
 
 func TestManifestSchema2EmbeddedDockerReferenceConflicts(t *testing.T) {
 	for _, m := range []genericManifest{
-		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json"),
+		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json", false),
 		manifestSchema2FromComponentsLikeFixture(nil),
 	} {
 		for _, name := range []string{"busybox", "example.com:5555/ns/repo:tag"} {
@@ -310,7 +314,7 @@ func TestManifestSchema2Inspect(t *testing.T) {
 
 func TestManifestSchema2UpdatedImageNeedsLayerDiffIDs(t *testing.T) {
 	for _, m := range []genericManifest{
-		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json"),
+		manifestSchema2FromFixture(t, unusedImageSource{}, "schema2.json", false),
 		manifestSchema2FromComponentsLikeFixture(nil),
 	} {
 		assert.False(t, m.UpdatedImageNeedsLayerDiffIDs(types.ManifestUpdateOptions{
@@ -438,7 +442,7 @@ func (d *memoryImageDest) Commit(ctx context.Context) error {
 
 func TestManifestSchema2UpdatedImage(t *testing.T) {
 	originalSrc := newSchema2ImageSource(t, "httpd:latest")
-	original := manifestSchema2FromFixture(t, originalSrc, "schema2.json")
+	original := manifestSchema2FromFixture(t, originalSrc, "schema2.json", false)
 
 	// LayerInfos:
 	layerInfos := append(original.LayerInfos()[1:], original.LayerInfos()[0])
@@ -490,7 +494,7 @@ func TestManifestSchema2UpdatedImage(t *testing.T) {
 	}
 
 	// m hasn’t been changed:
-	m2 := manifestSchema2FromFixture(t, originalSrc, "schema2.json")
+	m2 := manifestSchema2FromFixture(t, originalSrc, "schema2.json", false)
 	typedOriginal, ok := original.(*manifestSchema2)
 	require.True(t, ok)
 	typedM2, ok := m2.(*manifestSchema2)
@@ -500,7 +504,7 @@ func TestManifestSchema2UpdatedImage(t *testing.T) {
 
 func TestConvertToManifestOCI(t *testing.T) {
 	originalSrc := newSchema2ImageSource(t, "httpd-copy:latest")
-	original := manifestSchema2FromFixture(t, originalSrc, "schema2.json")
+	original := manifestSchema2FromFixture(t, originalSrc, "schema2.json", false)
 	res, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{
 		ManifestMIMEType: imgspecv1.MediaTypeImageManifest,
 	})
@@ -520,9 +524,35 @@ func TestConvertToManifestOCI(t *testing.T) {
 	assert.Equal(t, byHand, converted)
 }
 
+func TestConvertToManifestOCIAllMediaTypes(t *testing.T) {
+	originalSrc := newSchema2ImageSource(t, "httpd-copy:latest")
+	original := manifestSchema2FromFixture(t, originalSrc, "schema2-all-media-types.json", false)
+	res, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{
+		ManifestMIMEType: imgspecv1.MediaTypeImageManifest,
+	})
+	require.NoError(t, err)
+	convertedJSON, mt, err := res.Manifest(context.Background())
+	require.NoError(t, err)
+	assert.Equal(t, imgspecv1.MediaTypeImageManifest, mt)
+
+	byHandJSON, err := ioutil.ReadFile("fixtures/schema2-all-media-types-to-oci1.json")
+	require.NoError(t, err)
+	var converted, byHand map[string]interface{}
+	err = json.Unmarshal(byHandJSON, &byHand)
+	require.NoError(t, err)
+	err = json.Unmarshal(convertedJSON, &converted)
+	require.NoError(t, err)
+	assert.Equal(t, byHand, converted)
+}
+
+func TestConvertToOCIWithInvalidMIMEType(t *testing.T) {
+	originalSrc := newSchema2ImageSource(t, "httpd-copy:latest")
+	manifestSchema2FromFixture(t, originalSrc, "schema2-invalid-media-type.json", true)
+}
+
 func TestConvertToManifestSchema1(t *testing.T) {
 	originalSrc := newSchema2ImageSource(t, "httpd-copy:latest")
-	original := manifestSchema2FromFixture(t, originalSrc, "schema2.json")
+	original := manifestSchema2FromFixture(t, originalSrc, "schema2.json", false)
 	memoryDest := &memoryImageDest{ref: originalSrc.ref}
 	res, err := original.UpdatedImage(context.Background(), types.ManifestUpdateOptions{
 		ManifestMIMEType: manifest.DockerV2Schema1SignedMediaType,

image/fixtures/oci1-all-media-types-to-schema2.json

@@ -0,0 +1,41 @@
+{
+    "schemaVersion": 2,
+    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+    "config": {
+       "mediaType": "application/vnd.docker.container.image.v1+json",
+       "size": 4651,
+       "digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.zstd",
+            "size": 150,
+            "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+            "size": 152,
+            "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar",
+            "size": 11739507,
+            "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+            "size": 8841833,
+            "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
+        },
+        {
+            "mediaType": "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip",
+            "size": 291,
+            "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
+        }
+    ]
+}

image/fixtures/oci1-all-media-types.json

@@ -0,0 +1,40 @@
+{
+    "schemaVersion": 2,
+    "config": {
+        "mediaType": "application/vnd.oci.image.config.v1+json",
+        "size": 4651,
+        "digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar+zstd",
+            "size": 150,
+            "digest": "sha256:1bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+            "size": 152,
+            "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar",
+            "size": 11739507,
+            "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+            "size": 8841833,
+            "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+            "size": 291,
+            "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
+        }
+    ]
+}

image/fixtures/oci1-invalid-media-type.json

@@ -0,0 +1,15 @@
+{
+    "schemaVersion": 2,
+    "config": {
+        "mediaType": "application/vnd.oci.image.config.v1+json",
+        "size": 5940,
+        "digest": "sha256:9ca4bda0a6b3727a6ffcc43e981cad0f24e2ec79d338f6ba325b4dfd0756fb8f"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar+invalid-suffix",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        }
+    ]
+}

image/fixtures/oci1-to-schema2.json

@@ -34,4 +34,4 @@
          "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
       }
    ]
-}
+}

image/fixtures/schema2-all-media-types-to-oci1.json

@@ -0,0 +1,35 @@
+{
+    "schemaVersion": 2,
+    "config": {
+        "mediaType": "application/vnd.oci.image.config.v1+json",
+        "size": 4651,
+        "digest": "sha256:a13a0762ab7bed51a1b49adec0a702b1cd99294fd460a025b465bcfb7b152745"
+    },
+    "layers": [
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar",
+            "size": 51354364,
+            "digest": "sha256:6a5a5368e0c2d3e5909184fa28ddfd56072e7ff3ee9a945876f7eee5896ef5bb"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+            "size": 152,
+            "digest": "sha256:2bbf5d58d24c47512e234a5623474acf65ae00d4d1414272a893204f44cc680c"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar",
+            "size": 11739507,
+            "digest": "sha256:8f5dc8a4b12c307ac84de90cdd9a7f3915d1be04c9388868ca118831099c67a9"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+            "size": 8841833,
+            "digest": "sha256:bbd6b22eb11afce63cc76f6bc41042d99f10d6024c96b655dafba930b8d25909"
+        },
+        {
+            "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
+            "size": 291,
+            "digest": "sha256:960e52ecf8200cbd84e70eb2ad8678f4367e50d14357021872c10fa3fc5935fa"
+        }
+    ]
+}