IR-114: Adding support for OCI schema

go.mod

@@ -25,6 +25,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822
 	github.com/ncw/swift v1.0.49 // indirect
 	github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2
+	github.com/opencontainers/image-spec v1.0.1
 	github.com/opencontainers/runc v1.0.0-rc5.0.20180920170208-00dc70017d22 // indirect
 	github.com/openshift/api v0.0.0-20200827090112-c05698d102cf
 	github.com/openshift/client-go v0.0.0-20200827190008-3062137373b5

pkg/dockerregistry/server/manifesthandler/manifesthandler.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/docker/distribution"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/opencontainers/go-digest"
@@ -42,6 +43,8 @@ func NewManifestHandler(serverAddr string, blobStore distribution.BlobStore, man
 		return &manifestSchema1Handler{serverAddr: serverAddr, blobStore: blobStore, manifest: t}, nil
 	case *schema2.DeserializedManifest:
 		return &manifestSchema2Handler{blobStore: blobStore, manifest: t}, nil
+	case *ocischema.DeserializedManifest:
+		return &manifestOCIHandler{blobStore: blobStore, manifest: t}, nil
 	default:
 		return nil, fmt.Errorf("unsupported manifest type %T", manifest)
 	}

pkg/dockerregistry/server/manifesthandler/manifestocihandler.go

@@ -0,0 +1,136 @@
+package manifesthandler
+
+import (
+	"context"
+	"time"
+
+	"github.com/docker/distribution"
+	dcontext "github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest/ocischema"
+	"github.com/opencontainers/go-digest"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	imageapiv1 "github.com/openshift/api/image/v1"
+	imageapi "github.com/openshift/image-registry/pkg/origin-common/image/apis/image"
+)
+
+type manifestOCIHandler struct {
+	blobStore    distribution.BlobStore
+	manifest     *ocischema.DeserializedManifest
+	cachedConfig []byte
+}
+
+var _ ManifestHandler = &manifestOCIHandler{}
+
+func (h *manifestOCIHandler) Config(ctx context.Context) ([]byte, error) {
+	if h.cachedConfig == nil {
+		blob, err := h.blobStore.Get(ctx, h.manifest.Config.Digest)
+		if err != nil {
+			dcontext.GetLogger(ctx).Errorf("failed to get manifest config: %v", err)
+			return nil, err
+		}
+		h.cachedConfig = blob
+	}
+
+	return h.cachedConfig, nil
+}
+
+func (h *manifestOCIHandler) Digest() (digest.Digest, error) {
+	_, p, err := h.manifest.Payload()
+	if err != nil {
+		return "", err
+	}
+	return digest.FromBytes(p), nil
+}
+
+func (h *manifestOCIHandler) Manifest() distribution.Manifest {
+	return h.manifest
+}
+
+func (h *manifestOCIHandler) Layers(ctx context.Context) (string, []imageapiv1.ImageLayer, error) {
+	layers := make([]imageapiv1.ImageLayer, len(h.manifest.Layers))
+	for i, layer := range h.manifest.Layers {
+		layers[i].Name = layer.Digest.String()
+		layers[i].LayerSize = layer.Size
+		layers[i].MediaType = layer.MediaType
+	}
+	return imageapi.DockerImageLayersOrderAscending, layers, nil
+}
+
+func (h *manifestOCIHandler) Payload() (mediaType string, payload []byte, canonical []byte, err error) {
+	mt, p, err := h.manifest.Payload()
+	return mt, p, p, err
+}
+
+func (h *manifestOCIHandler) verifyLayer(ctx context.Context, fsLayer distribution.Descriptor) error {
+	// https://bugzilla.redhat.com/show_bug.cgi?id=1745743
+	// AWS S3 (and potentially other object stores) only have eventual
+	// consistency guarantees. Stat can fail here if an image layer was
+	// recently pushed. In the event Stat returns `ErrBlobUnknown`, retry
+	// up to 3 seconds.
+	var desc distribution.Descriptor
+	if err := wait.ExponentialBackoff(
+		wait.Backoff{
+			Duration: 100 * time.Millisecond,
+			Factor:   2,
+			Steps:    6,
+		},
+		func() (done bool, err error) {
+			desc, err = h.blobStore.Stat(ctx, fsLayer.Digest)
+			switch {
+			case err == nil:
+				return true, nil
+			case err == distribution.ErrBlobUnknown:
+				return false, nil
+			default:
+				return true, err
+			}
+		},
+	); err != nil {
+		if err == wait.ErrWaitTimeout {
+			return distribution.ErrBlobUnknown
+		}
+		return err
+	}
+
+	if fsLayer.Size != desc.Size {
+		return ErrManifestBlobBadSize{
+			Digest:         fsLayer.Digest,
+			ActualSize:     desc.Size,
+			SizeInManifest: fsLayer.Size,
+		}
+	}
+
+	return nil
+}
+
+func (h *manifestOCIHandler) Verify(ctx context.Context, skipDependencyVerification bool) error {
+	var errs distribution.ErrManifestVerification
+
+	if skipDependencyVerification {
+		return nil
+	}
+
+	// we want to verify that referenced blobs exist locally or accessible over
+	// pullthroughBlobStore. The base image of this image can be remote repository
+	// and since we use pullthroughBlobStore all the layer existence checks will be
+	// successful. This means that the docker client will not attempt to send them
+	// to us as it will assume that the registry has them.
+
+	for _, fsLayer := range h.manifest.References() {
+		if err := h.verifyLayer(ctx, fsLayer); err != nil {
+			if err != distribution.ErrBlobUnknown {
+				errs = append(errs, err)
+				continue
+			}
+
+			// On error here, we always append unknown blob errors.
+			errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest})
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}

pkg/testutil/manifests.go

@@ -10,11 +10,13 @@ import (
 
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest"
+	"github.com/docker/distribution/manifest/ocischema"
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/docker/distribution/registry/client/auth"
 	"github.com/docker/libtrust"
 	"github.com/opencontainers/go-digest"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
@@ -24,13 +26,14 @@ import (
 	util "github.com/openshift/image-registry/pkg/origin-common/util"
 )
 
-type ManifestSchemaVersion int
+type ManifestSchemaVersion string
 
 type ConfigPayload []byte
 
 const (
-	ManifestSchema1 ManifestSchemaVersion = 1
-	ManifestSchema2 ManifestSchemaVersion = 2
+	ManifestSchema1   ManifestSchemaVersion = "v1"
+	ManifestSchema2   ManifestSchemaVersion = "v2"
+	ManifestSchemaOCI ManifestSchemaVersion = "oci"
 )
 
 // MakeSchema1Manifest constructs a schema 1 manifest from a given list of digests and returns
@@ -87,12 +90,30 @@ func MakeSchema2Manifest(config distribution.Descriptor, layers []distribution.D
 	return manifest, nil
 }
 
+// MakeOCISchemaManifest constructs an OCI schema manifest from a given list of digests and returns
+// the digest of the manifest
+func MakeOCISchemaManifest(config distribution.Descriptor, layers []distribution.Descriptor) (distribution.Manifest, error) {
+	m := ocischema.Manifest{
+		Versioned: ocischema.SchemaVersion,
+		Config:    config,
+		Layers:    make([]distribution.Descriptor, 0, len(layers)),
+	}
+	m.Config.MediaType = v1.MediaTypeImageConfig
+
+	for _, layer := range layers {
+		layer.MediaType = v1.MediaTypeImageLayer
+		m.Layers = append(m.Layers, layer)
+	}
+
+	return ocischema.FromStruct(m)
+}
+
 // CanonicalManifest returns m in its canonical representation.
 func CanonicalManifest(m distribution.Manifest) ([]byte, error) {
 	switch m := m.(type) {
 	case *schema1.SignedManifest:
 		return m.Canonical, nil
-	case *schema2.DeserializedManifest:
+	case *schema2.DeserializedManifest, *ocischema.DeserializedManifest:
 		_, payload, err := m.Payload()
 		if err != nil {
 			return nil, err
@@ -182,8 +203,22 @@ func CreateAndUploadTestManifest(
 			return "", "", "", nil, fmt.Errorf("failed to make manifest schema 2: %v", err)
 		}
 		manifestConfig = string(cfgPayload)
+	case ManifestSchemaOCI:
+		cfgPayload, cfgDesc, err := MakeManifestConfig()
+		if err != nil {
+			return "", "", "", nil, err
+		}
+		err = UploadBlob(ctx, repo, cfgDesc, cfgPayload)
+		if err != nil {
+			return "", "", "", nil, fmt.Errorf("failed to upload manifest config of schema 2: %v", err)
+		}
+		manifest, err = MakeOCISchemaManifest(cfgDesc, layerDescriptors)
+		if err != nil {
+			return "", "", "", nil, fmt.Errorf("failed to make manifest schema 2: %v", err)
+		}
+		manifestConfig = string(cfgPayload)
 	default:
-		return "", "", "", nil, fmt.Errorf("unsupported manifest version %d", schemaVersion)
+		return "", "", "", nil, fmt.Errorf("unsupported manifest version %s", schemaVersion)
 	}
 
 	canonicalBytes, err := CanonicalManifest(manifest)

test/integration/oci/oci_test.go

@@ -0,0 +1,66 @@
+package integration
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"testing"
+	"time"
+
+	imageclientv1 "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/openshift/image-registry/pkg/testframework"
+	"github.com/openshift/image-registry/pkg/testutil"
+)
+
+func TestOCIPush(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
+	defer cancel()
+
+	master := testframework.NewMaster(t)
+	defer master.Close()
+	registry := master.StartRegistry(t)
+	defer registry.Close()
+
+	namespace := "oci-integration-test"
+	isname := "imagestream"
+	testuser := master.CreateUser("testuser", "testp@ssw0rd")
+	proj := master.CreateProject(namespace, testuser.Name)
+
+	regURL, err := url.Parse(registry.BaseURL())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dgst, _, _, _, err := testutil.CreateAndUploadTestManifest(
+		ctx,
+		testutil.ManifestSchemaOCI,
+		5,
+		regURL,
+		testutil.NewBasicCredentialStore(testuser.Name, testuser.Token),
+		fmt.Sprintf("%s/%s", proj.Name, isname),
+		"latest",
+	)
+	if err != nil {
+		t.Errorf("error uploading manifest: %s", err)
+	}
+
+	imgcli := imageclientv1.NewForConfigOrDie(testuser.KubeConfig())
+	is, err := imgcli.ImageStreams(namespace).Get(
+		ctx, isname, metav1.GetOptions{})
+	if err != nil {
+		t.Errorf("error getting image stream: %s", err)
+	}
+	if len(is.Status.Tags) != 1 {
+		t.Fatal("no tag found in image stream")
+	}
+
+	if is.Status.Tags[0].Items[0].Image != dgst.String() {
+		t.Errorf(
+			"expecting image digest %s, received %s instead",
+			dgst,
+			is.Status.Tags[0].Items[0].Image,
+		)
+	}
+}