From 0bffef61fb77d283ef58242f1e234eab64b41683 Mon Sep 17 00:00:00 2001 From: Jeff Carter Date: Fri, 15 May 2026 16:40:12 -0400 Subject: [PATCH] Add package for calculating OCI digests. This unblocks copying in OCI model types. --- README.md | 6 +- go.mod | 2 - go.sum | 4 - interface.go | 13 - models.go | 222 ++++++++++++++++++ ociclient/auth_test.go | 26 +- ociclient/badname_test.go | 5 +- ociclient/client.go | 39 +-- ociclient/deleter.go | 4 +- ociclient/error_test.go | 24 +- ociclient/lister.go | 11 +- ociclient/reader.go | 28 ++- ociclient/referrers_test.go | 33 +-- ociclient/referrerstag_test.go | 19 +- ociclient/writer.go | 28 +-- ocidigest/README.md | 350 +++++++++++++++++++++++++++ ocidigest/algorithm.go | 199 ++++++++++++++++ ocidigest/digest.go | 147 ++++++++++++ ocidigest/digest_test.go | 417 +++++++++++++++++++++++++++++++++ ocidigest/digester.go | 84 +++++++ ocidigest/doc.go | 2 + ocidigest/encoding.go | 58 +++++ ocidigest/errors.go | 18 ++ ocidigest/multidigester.go | 218 +++++++++++++++++ ocidigest/reader.go | 62 +++++ ocidigest/state.go | 82 +++++++ ocidigest/verifier.go | 140 +++++++++++ ocidigest/writer.go | 63 +++++ ocifilter/immutable.go | 6 +- ocifilter/select_test.go | 26 +- ocifilter/sub_test.go | 46 ++-- ocilarge/download.go | 20 +- ocilarge/upload.go | 44 +++- ocimem/blob.go | 4 +- ocimem/check_test.go | 242 +++++++++---------- ocimem/desciter.go | 94 +++----- ocimem/lister.go | 2 +- ocimem/registry.go | 12 +- ocimem/writer.go | 13 +- ociref/reference.go | 26 +- ociref/reference_test.go | 23 +- ociserver/deleter.go | 14 +- ociserver/lister.go | 14 +- ociserver/mediatype.go | 10 +- ociserver/proxy_test.go | 13 +- ociserver/reader.go | 46 +++- ociserver/registry.go | 7 +- ociserver/registry_test.go | 4 +- ociserver/writer.go | 58 +++-- ocitest/ocitest.go | 50 ++-- ociunify/lister.go | 2 +- 51 files changed, 2632 insertions(+), 448 deletions(-) create mode 100644 models.go create mode 100644 ocidigest/README.md create mode 100644 ocidigest/algorithm.go create mode 100644 ocidigest/digest.go create mode 100644 ocidigest/digest_test.go create mode 100644 ocidigest/digester.go create mode 100644 ocidigest/doc.go create mode 100644 ocidigest/encoding.go create mode 100644 ocidigest/errors.go create mode 100644 ocidigest/multidigester.go create mode 100644 ocidigest/reader.go create mode 100644 ocidigest/state.go create mode 100644 ocidigest/verifier.go create mode 100644 ocidigest/writer.go diff --git a/README.md b/README.md index ad82af9..5094e14 100644 --- a/README.md +++ b/README.md @@ -87,9 +87,9 @@ import ( "encoding/json" "fmt" + "github.com/docker/oci" "github.com/docker/oci/ociauth" "github.com/docker/oci/ociclient" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) func main() { @@ -117,7 +117,7 @@ func main() { fmt.Printf("media type: %s\n", r.Descriptor().MediaType) fmt.Printf("digest: %s\n", r.Descriptor().Digest) - var manifest ocispec.Manifest + var manifest oci.IndexOrManifest if err := json.NewDecoder(r).Decode(&manifest); err != nil { panic(err) } @@ -196,4 +196,4 @@ func main() { handler := ociserver.New(backend, nil) http.ListenAndServe(":5000", handler) } -``` \ No newline at end of file +``` diff --git a/go.mod b/go.mod index eefcace..05d3499 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,6 @@ module github.com/docker/oci go 1.25.0 require ( - github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.1 github.com/rogpeppe/go-internal v1.14.1 github.com/stretchr/testify v1.11.1 ) diff --git a/go.sum b/go.sum index be5ce8d..0ea362a 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= -github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= diff --git a/interface.go b/interface.go index ee2903c..6e9362b 100644 --- a/interface.go +++ b/interface.go @@ -60,9 +60,6 @@ import ( "context" "io" "iter" - - "github.com/docker/oci/ociref" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // Interface defines a generic interface to a single OCI registry. @@ -83,16 +80,6 @@ type ReadWriter interface { Writer } -// Type aliases for commonly used OCI types. -type ( - // Digest is a content-addressable digest. It is an alias for [ociref.Digest]. - Digest = ociref.Digest - // Descriptor describes the disposition of targeted content. It is an alias for [ocispec.Descriptor]. - Descriptor = ocispec.Descriptor - // Manifest provides the `application/vnd.oci.image.manifest.v1+json` mediatype structure. It is an alias for [ocispec.Manifest]. - Manifest = ocispec.Manifest -) - // Reader defines registry operations that read blobs, manifests and tags. type Reader interface { // GetBlob returns the content of the blob with the given digest. diff --git a/models.go b/models.go new file mode 100644 index 0000000..d8da178 --- /dev/null +++ b/models.go @@ -0,0 +1,222 @@ +// Copyright 2023 CUE Labs AG +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oci + +import ( + "encoding/json" + "fmt" + + "github.com/docker/oci/ocidigest" +) + +const ( + // MediaTypeImageIndex specifies the media type for an image index. + MediaTypeImageIndex = "application/vnd.oci.image.index.v1+json" + // MediaTypeImageManifest specifies the media type for an image manifest. + MediaTypeImageManifest = "application/vnd.oci.image.manifest.v1+json" + // MediaTypeDockerManifestList is the media type Docker used to use for an index + MediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" + // MediaTypeDockerManifest is the mediat type Docker used to use for a manifest + MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json" + + // MediaTypeImageConfig specifies the media type for the image configuration. + MediaTypeImageConfig = "application/vnd.oci.image.config.v1+json" + + // MaxMediaTypeLen is the maximum allowed length for any mediaType field. + MaxMediaTypeLen = 255 + // MaxArtifactTypeLen is the maximum allowed length for any artifactType field. + MaxArtifactTypeLen = 255 + // MaxAnnotationKeyLen is the maximum allowed length for an annotation key. + MaxAnnotationKeyLen = 255 +) + +// Digest is a content-addressable digest. +type Digest = ocidigest.Digest + +// Descriptor describes the disposition of targeted content. +type Descriptor struct { + // MediaType is the media type of the object this schema refers to. + MediaType string `json:"mediaType"` + + // Digest is the digest of the targeted content. + Digest Digest `json:"digest"` + + // Size specifies the size in bytes of the blob. + Size int64 `json:"size"` + + // URLs specifies a list of URLs from which this object MAY be downloaded. + URLs []string `json:"urls,omitempty"` + + // Annotations contains arbitrary metadata relating to the targeted content. + Annotations map[string]string `json:"annotations,omitempty"` + + // Data is an embedding of the targeted content. + Data []byte `json:"data,omitempty"` + + // Platform describes the platform which the image in the manifest runs on. + Platform *Platform `json:"platform,omitempty"` + + // ArtifactType is the IANA media type of this artifact. + ArtifactType string `json:"artifactType,omitempty"` +} + +// Platform describes the platform which the image in the manifest runs on. +type Platform struct { + // Architecture field specifies the CPU architecture, for example amd64 or ppc64le. + Architecture string `json:"architecture"` + + // OS specifies the operating system, for example linux or windows. + OS string `json:"os"` + + // OSVersion is an optional field specifying the operating system version. + OSVersion string `json:"os.version,omitempty"` + + // OSFeatures is an optional field specifying an array of strings listing required OS features. + OSFeatures []string `json:"os.features,omitempty"` + + // Variant is an optional field specifying a variant of the CPU. + Variant string `json:"variant,omitempty"` +} + +// IndexOrManifest parses the required fields out of a manifest json file. It handles indexes and manifests. +type IndexOrManifest struct { + SchemaVersion int `json:"schemaVersion"` + + // MediaType specifies the type of this document data structure e.g. `application/vnd.oci.image.manifest.v1+json` // TODO: add validation... if index, make sure it has manifests instead of layers? + MediaType string `json:"mediaType,omitempty"` + + // ArtifactType specifies the IANA media type of artifact when the manifest is used for an artifact. + ArtifactType string `json:"artifactType,omitempty"` + + // Manifests references platform specific manifests. + Manifests []Descriptor `json:"manifests"` + + // Config references a configuration object for a container, by digest. + // The referenced configuration object is a JSON blob that the runtime uses to set up the container. + Config *Descriptor `json:"config"` + + // Layers is an indexed list of layers referenced by the manifest. + Layers []Descriptor `json:"layers"` + + // Subject is an optional link from the image manifest to another manifest forming an association between the image manifest and the other manifest. + Subject *Descriptor `json:"subject,omitempty"` + + // Annotations contains arbitrary metadata for the image manifest. + Annotations map[string]string `json:"annotations,omitempty"` +} + +// Validate checks the manifest for structural correctness and field length limits. +func (m IndexOrManifest) Validate() error { + if m.SchemaVersion == 1 { + return fmt.Errorf("schema version 1 (Docker V1) manifests are not supported") + } + switch m.MediaType { + case MediaTypeImageIndex, MediaTypeDockerManifestList: + if m.Config != nil { + return fmt.Errorf("config not supported on index") + } + if len(m.Layers) > 0 { + return fmt.Errorf("layers not supported on index") + } + case MediaTypeImageManifest, MediaTypeDockerManifest: + if len(m.Manifests) > 0 { + return fmt.Errorf("manifests field not supported on manifest") + } + if m.Config == nil { + return fmt.Errorf("missing config") + } + } + if len(m.MediaType) > MaxMediaTypeLen { + return fmt.Errorf("mediaType exceeds maximum length of %d", MaxMediaTypeLen) + } + if len(m.ArtifactType) > MaxArtifactTypeLen { + return fmt.Errorf("artifactType exceeds maximum length of %d", MaxArtifactTypeLen) + } + for k := range m.Annotations { + if len(k) > MaxAnnotationKeyLen { + return fmt.Errorf("annotation key exceeds maximum length of %d", MaxAnnotationKeyLen) + } + } + for i, d := range m.Manifests { + if err := validateDescriptor("manifests", i, d); err != nil { + return err + } + } + for i, d := range m.Layers { + if err := validateDescriptor("layers", i, d); err != nil { + return err + } + } + return nil +} + +// MarshalJSON validates m before encoding it as JSON. +func (m IndexOrManifest) MarshalJSON() ([]byte, error) { + if err := m.Validate(); err != nil { + return nil, err + } + type common struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Subject *Descriptor `json:"subject,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + } + c := common{ + SchemaVersion: m.SchemaVersion, + MediaType: m.MediaType, + ArtifactType: m.ArtifactType, + Subject: m.Subject, + Annotations: m.Annotations, + } + switch m.MediaType { + case MediaTypeImageIndex, MediaTypeDockerManifestList: + return json.Marshal(struct { + common + Manifests []Descriptor `json:"manifests"` + }{ + common: c, + Manifests: m.Manifests, + }) + case MediaTypeImageManifest, MediaTypeDockerManifest: + return json.Marshal(struct { + common + Config *Descriptor `json:"config"` + Layers []Descriptor `json:"layers"` + }{ + common: c, + Config: m.Config, + Layers: m.Layers, + }) + default: + type indexOrManifest IndexOrManifest + return json.Marshal(indexOrManifest(m)) + } +} + +func validateDescriptor(field string, i int, d Descriptor) error { + if len(d.MediaType) > MaxMediaTypeLen { + return fmt.Errorf("%s[%d].mediaType exceeds maximum length of %d", field, i, MaxMediaTypeLen) + } + if len(d.ArtifactType) > MaxArtifactTypeLen { + return fmt.Errorf("%s[%d].artifactType exceeds maximum length of %d", field, i, MaxArtifactTypeLen) + } + for k := range d.Annotations { + if len(k) > MaxAnnotationKeyLen { + return fmt.Errorf("%s[%d] annotation key exceeds maximum length of %d", field, i, MaxAnnotationKeyLen) + } + } + return nil +} diff --git a/ociclient/auth_test.go b/ociclient/auth_test.go index fc5932f..21e636d 100644 --- a/ociclient/auth_test.go +++ b/ociclient/auth_test.go @@ -10,13 +10,15 @@ import ( "github.com/docker/oci" "github.com/docker/oci/ociauth" + "github.com/docker/oci/ocidigest" "github.com/docker/oci/ocimem" "github.com/docker/oci/ociserver" - "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var testDigest = ocidigest.FromBytes([]byte("test")) + func TestAuthScopes(t *testing.T) { // Test that we're passing the expected authorization scopes to the various parts of the API. @@ -32,22 +34,22 @@ func TestAuthScopes(t *testing.T) { } assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { - r.GetBlob(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + r.GetBlob(ctx, "foo/bar", testDigest) }) assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { - r.GetBlobRange(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 100, 200) + r.GetBlobRange(ctx, "foo/bar", testDigest, 100, 200) }) assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { - r.GetManifest(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + r.GetManifest(ctx, "foo/bar", testDigest) }) assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { r.GetTag(ctx, "foo/bar", "sometag") }) assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { - r.ResolveBlob(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + r.ResolveBlob(ctx, "foo/bar", testDigest) }) assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { - r.ResolveManifest(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + r.ResolveManifest(ctx, "foo/bar", testDigest) }) assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { r.ResolveTag(ctx, "foo/bar", "sometag") @@ -55,7 +57,7 @@ func TestAuthScopes(t *testing.T) { assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { r.PushBlob(ctx, "foo/bar", oci.Descriptor{ MediaType: "application/json", - Digest: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: testDigest, Size: 3, }, strings.NewReader("foo")) }) @@ -69,11 +71,11 @@ func TestAuthScopes(t *testing.T) { w, err = r.PushBlobChunkedResume(ctx, "foo/bar", id, 3, 0) require.NoError(t, err) w.Write([]byte("bar")) - _, err = w.Commit(digest.FromString("foobar")) + _, err = w.Commit(ocidigest.FromBytes([]byte("foobar"))) require.NoError(t, err) }) assertScope("repository:x/y:pull repository:z/w:push", func(ctx context.Context, r oci.Interface) { - r.MountBlob(ctx, "x/y", "z/w", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + r.MountBlob(ctx, "x/y", "z/w", testDigest) }) assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { r.PushManifest(ctx, "foo/bar", []byte("something"), "application/json", &oci.PushManifestParameters{ @@ -81,10 +83,10 @@ func TestAuthScopes(t *testing.T) { }) }) assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { - r.DeleteBlob(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + r.DeleteBlob(ctx, "foo/bar", testDigest) }) assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { - r.DeleteManifest(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + r.DeleteManifest(ctx, "foo/bar", testDigest) }) assertScope("repository:foo/bar:push", func(ctx context.Context, r oci.Interface) { r.DeleteTag(ctx, "foo/bar", "sometag") @@ -96,7 +98,7 @@ func TestAuthScopes(t *testing.T) { oci.All(r.Tags(ctx, "foo/bar", nil)) }) assertScope("repository:foo/bar:pull", func(ctx context.Context, r oci.Interface) { - oci.All(r.Referrers(ctx, "foo/bar", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", nil)) + oci.All(r.Referrers(ctx, "foo/bar", testDigest, nil)) }) } diff --git a/ociclient/badname_test.go b/ociclient/badname_test.go index b385157..77f228b 100644 --- a/ociclient/badname_test.go +++ b/ociclient/badname_test.go @@ -6,6 +6,7 @@ import ( "net/http" "testing" + "github.com/docker/oci/ocidigest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -17,10 +18,8 @@ func TestBadRepoName(t *testing.T) { Transport: noTransport{}, }) require.NoError(t, err) - _, err = r.GetBlob(ctx, "Invalid--Repo", "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + _, err = r.GetBlob(ctx, "Invalid--Repo", ocidigest.FromBytes(nil)) assert.Regexp(t, "invalid OCI request: name invalid: invalid repository name", err.Error()) - _, err = r.GetBlob(ctx, "okrepo", "bad-digest") - assert.Regexp(t, "invalid OCI request: digest invalid: badly formed digest", err.Error()) _, err = r.ResolveTag(ctx, "okrepo", "bad-Tag!") assert.Regexp(t, "invalid OCI request: 404 Not Found: page not found", err.Error()) } diff --git a/ociclient/client.go b/ociclient/client.go index 3580028..7e1e7db 100644 --- a/ociclient/client.go +++ b/ociclient/client.go @@ -20,7 +20,6 @@ import ( "bytes" "context" "fmt" - "hash" "io" "log" "net/http" @@ -31,11 +30,10 @@ import ( "sync/atomic" "github.com/docker/oci" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/docker/oci/internal/ocirequest" "github.com/docker/oci/ociauth" + "github.com/docker/oci/ocidigest" "github.com/docker/oci/ociref" ) @@ -135,7 +133,7 @@ const ( // // Note: this implies that the Digest field will be empty if there is no // digest in the response and knownDigest is empty. -func descriptorFromResponse(resp *http.Response, knownDigest digest.Digest, require descriptorRequired) (oci.Descriptor, error) { +func descriptorFromResponse(resp *http.Response, knownDigest oci.Digest, require descriptorRequired) (oci.Descriptor, error) { contentType := resp.Header.Get("Content-Type") if contentType == "" { contentType = "application/octet-stream" @@ -163,15 +161,18 @@ func descriptorFromResponse(resp *http.Response, knownDigest digest.Digest, requ size = resp.ContentLength } } - digest := digest.Digest(resp.Header.Get("Docker-Content-Digest")) - if digest != "" { - if !ociref.IsValidDigest(string(digest)) { + digest, err := ocidigest.Parse(resp.Header.Get("Docker-Content-Digest")) + if err != nil { + return oci.Descriptor{}, fmt.Errorf("bad digest %q found in response: %v", resp.Header.Get("Docker-Content-Digest"), err) + } + if !digest.IsZero() { + if !ociref.IsValidDigest(digest.String()) { return oci.Descriptor{}, fmt.Errorf("bad digest %q found in response", digest) } } else { digest = knownDigest } - if (require&requireDigest) != 0 && digest == "" { + if (require&requireDigest) != 0 && digest.IsZero() { return oci.Descriptor{}, fmt.Errorf("no digest found in response") } return oci.Descriptor{ @@ -182,9 +183,10 @@ func descriptorFromResponse(resp *http.Response, knownDigest digest.Digest, requ } func newBlobReader(r io.ReadCloser, desc oci.Descriptor) *blobReader { + digester, _ := desc.Digest.Algorithm().New() return &blobReader{ r: r, - digester: desc.Digest.Algorithm().Hash(), + digester: digester, desc: desc, verify: true, } @@ -199,7 +201,7 @@ func newBlobReaderUnverified(r io.ReadCloser, desc oci.Descriptor) *blobReader { type blobReader struct { r io.ReadCloser n int64 - digester hash.Hash + digester ocidigest.Digester desc oci.Descriptor verify bool } @@ -211,7 +213,9 @@ func (r *blobReader) Descriptor() oci.Descriptor { func (r *blobReader) Read(buf []byte) (int, error) { n, err := r.r.Read(buf) r.n += int64(n) - r.digester.Write(buf[:n]) + if _, writeErr := r.digester.Write(buf[:n]); writeErr != nil { + return n, writeErr + } if err == nil { if r.n > r.desc.Size { // Fail early when the blob is too big; we can do that even @@ -229,7 +233,10 @@ func (r *blobReader) Read(buf []byte) (int, error) { if r.n != r.desc.Size { return n, fmt.Errorf("blob size mismatch (%d/%d): %w", r.n, r.desc.Size, oci.ErrSizeInvalid) } - gotDigest := digest.NewDigest(r.desc.Digest.Algorithm(), r.digester) + gotDigest, err := r.digester.Digest() + if err != nil { + return n, err + } if gotDigest != r.desc.Digest { return n, fmt.Errorf("digest mismatch when reading blob") } @@ -242,12 +249,12 @@ func (r *blobReader) Close() error { // TODO make this list configurable. var knownManifestMediaTypes = []string{ - ocispec.MediaTypeImageManifest, - ocispec.MediaTypeImageIndex, + oci.MediaTypeImageManifest, + oci.MediaTypeImageIndex, "application/vnd.oci.artifact.manifest.v1+json", // deprecated. "application/vnd.docker.distribution.manifest.v1+json", - "application/vnd.docker.distribution.manifest.v2+json", - "application/vnd.docker.distribution.manifest.list.v2+json", + oci.MediaTypeDockerManifest, + oci.MediaTypeDockerManifestList, // Technically this wildcard should be sufficient, but it isn't // recognized by some registries. "*/*", diff --git a/ociclient/deleter.go b/ociclient/deleter.go index 1ad203d..2dc7a37 100644 --- a/ociclient/deleter.go +++ b/ociclient/deleter.go @@ -26,7 +26,7 @@ func (c *client) DeleteBlob(ctx context.Context, repoName string, digest oci.Dig return c.delete(ctx, &ocirequest.Request{ Kind: ocirequest.ReqBlobDelete, Repo: repoName, - Digest: string(digest), + Digest: digest.String(), }) } @@ -34,7 +34,7 @@ func (c *client) DeleteManifest(ctx context.Context, repoName string, digest oci return c.delete(ctx, &ocirequest.Request{ Kind: ocirequest.ReqManifestDelete, Repo: repoName, - Digest: string(digest), + Digest: digest.String(), }) } diff --git a/ociclient/error_test.go b/ociclient/error_test.go index 310322a..be7b581 100644 --- a/ociclient/error_test.go +++ b/ociclient/error_test.go @@ -10,8 +10,8 @@ import ( "testing" "github.com/docker/oci" + "github.com/docker/oci/ocidigest" "github.com/docker/oci/ociserver" - "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -64,21 +64,21 @@ func TestNonJSONErrorResponse(t *testing.T) { require.Equal(t, http.StatusTeapot, herr.StatusCode()) } assertStatusCode(func(ctx context.Context, r oci.Interface) error { - rd, err := r.GetBlob(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + rd, err := r.GetBlob(ctx, "foo/read", testDigest) if rd != nil { rd.Close() } return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - rd, err := r.GetBlobRange(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 100, 200) + rd, err := r.GetBlobRange(ctx, "foo/read", testDigest, 100, 200) if rd != nil { rd.Close() } return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - rd, err := r.GetManifest(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + rd, err := r.GetManifest(ctx, "foo/read", testDigest) if rd != nil { rd.Close() } @@ -92,11 +92,11 @@ func TestNonJSONErrorResponse(t *testing.T) { return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - _, err := r.ResolveBlob(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + _, err := r.ResolveBlob(ctx, "foo/read", testDigest) return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - _, err := r.ResolveManifest(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + _, err := r.ResolveManifest(ctx, "foo/read", testDigest) return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { @@ -106,7 +106,7 @@ func TestNonJSONErrorResponse(t *testing.T) { assertStatusCode(func(ctx context.Context, r oci.Interface) error { _, err := r.PushBlob(ctx, "foo/write", oci.Descriptor{ MediaType: "application/json", - Digest: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: testDigest, Size: 3, }, strings.NewReader("foo")) return err @@ -128,11 +128,11 @@ func TestNonJSONErrorResponse(t *testing.T) { if _, err := w.Write(data); err != nil { return err } - _, err = w.Commit(digest.FromBytes(data)) + _, err = w.Commit(ocidigest.FromBytes(data)) return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - _, err := r.MountBlob(ctx, "foo/read", "foo/write", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + _, err := r.MountBlob(ctx, "foo/read", "foo/write", testDigest) return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { @@ -142,10 +142,10 @@ func TestNonJSONErrorResponse(t *testing.T) { return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - return r.DeleteBlob(ctx, "foo/write", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + return r.DeleteBlob(ctx, "foo/write", testDigest) }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - return r.DeleteManifest(ctx, "foo/write", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + return r.DeleteManifest(ctx, "foo/write", testDigest) }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { return r.DeleteTag(ctx, "foo/write", "sometag") @@ -159,7 +159,7 @@ func TestNonJSONErrorResponse(t *testing.T) { return err }) assertStatusCode(func(ctx context.Context, r oci.Interface) error { - _, err := oci.All(r.Referrers(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", nil)) + _, err := oci.All(r.Referrers(ctx, "foo/read", testDigest, nil)) return err }) } diff --git a/ociclient/lister.go b/ociclient/lister.go index 18fb897..b18b6d0 100644 --- a/ociclient/lister.go +++ b/ociclient/lister.go @@ -26,7 +26,6 @@ import ( "strings" "github.com/docker/oci" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/docker/oci/internal/ocirequest" ) @@ -86,7 +85,7 @@ func (c *client) Referrers(ctx context.Context, repoName string, digest oci.Dige return pager(ctx, c, &ocirequest.Request{ Kind: ocirequest.ReqReferrersList, Repo: repoName, - Digest: string(digest), + Digest: digest.String(), ListN: -1, ArtifactType: artifactType, }, false, func(resp *http.Response) ([]oci.Descriptor, error) { @@ -114,10 +113,16 @@ func (c *client) Referrers(ctx context.Context, repoName string, digest oci.Dige if err != nil { return nil, err } - var referrersResponse ocispec.Index + var referrersResponse oci.IndexOrManifest if err := json.Unmarshal(data, &referrersResponse); err != nil { return nil, fmt.Errorf("cannot unmarshal referrers response: %v", err) } + if referrersResponse.MediaType == "" { + referrersResponse.MediaType = oci.MediaTypeImageIndex + } + if err := referrersResponse.Validate(); err != nil { + return nil, fmt.Errorf("invalid referrers response: %v", err) + } if artifactType == "" || resp.Header.Get("OCI-Filters-Applied") == "artifactType" { return referrersResponse.Manifests, nil } diff --git a/ociclient/reader.go b/ociclient/reader.go index 5195f4b..a9c1713 100644 --- a/ociclient/reader.go +++ b/ociclient/reader.go @@ -23,14 +23,14 @@ import ( "github.com/docker/oci" "github.com/docker/oci/internal/ocirequest" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" ) func (c *client) GetBlob(ctx context.Context, repo string, digest oci.Digest) (oci.BlobReader, error) { return c.read(ctx, &ocirequest.Request{ Kind: ocirequest.ReqBlobGet, Repo: repo, - Digest: string(digest), + Digest: digest.String(), }) } @@ -41,7 +41,7 @@ func (c *client) GetBlobRange(ctx context.Context, repo string, digest oci.Diges rreq := &ocirequest.Request{ Kind: ocirequest.ReqBlobGet, Repo: repo, - Digest: string(digest), + Digest: digest.String(), } req, err := newRequest(ctx, rreq, nil) if err != nil { @@ -60,7 +60,8 @@ func (c *client) GetBlobRange(ctx context.Context, repo string, digest oci.Diges // Fix that either by returning ErrUnsupported or by reading the whole // blob and returning only the required portion. defer closeOnError(&_err, resp.Body) - desc, err := descriptorFromResponse(resp, oci.Digest(rreq.Digest), requireSize) + knownDigest, _ := ocidigest.Parse(rreq.Digest) + desc, err := descriptorFromResponse(resp, knownDigest, requireSize) if err != nil { return nil, fmt.Errorf("invalid descriptor in response: %v", err) } @@ -71,7 +72,7 @@ func (c *client) ResolveBlob(ctx context.Context, repo string, digest oci.Digest return c.resolve(ctx, &ocirequest.Request{ Kind: ocirequest.ReqBlobHead, Repo: repo, - Digest: string(digest), + Digest: digest.String(), }) } @@ -79,7 +80,7 @@ func (c *client) ResolveManifest(ctx context.Context, repo string, digest oci.Di return c.resolve(ctx, &ocirequest.Request{ Kind: ocirequest.ReqManifestHead, Repo: repo, - Digest: string(digest), + Digest: digest.String(), }) } @@ -97,7 +98,8 @@ func (c *client) resolve(ctx context.Context, rreq *ocirequest.Request) (oci.Des return oci.Descriptor{}, err } resp.Body.Close() - desc, err := descriptorFromResponse(resp, oci.Digest(rreq.Digest), requireSize|requireDigest) + knownDigest, _ := ocidigest.Parse(rreq.Digest) + desc, err := descriptorFromResponse(resp, knownDigest, requireSize|requireDigest) if err != nil { return oci.Descriptor{}, fmt.Errorf("invalid descriptor in response: %v", err) } @@ -108,7 +110,7 @@ func (c *client) GetManifest(ctx context.Context, repo string, digest oci.Digest return c.read(ctx, &ocirequest.Request{ Kind: ocirequest.ReqManifestGet, Repo: repo, - Digest: string(digest), + Digest: digest.String(), }) } @@ -138,11 +140,12 @@ func (c *client) read(ctx context.Context, rreq *ocirequest.Request) (_ oci.Blob return nil, err } defer closeOnError(&_err, resp.Body) - desc, err := descriptorFromResponse(resp, oci.Digest(rreq.Digest), requireSize) + knownDigest, _ := ocidigest.Parse(rreq.Digest) + desc, err := descriptorFromResponse(resp, knownDigest, requireSize) if err != nil { return nil, fmt.Errorf("invalid descriptor in response: %v", err) } - if desc.Digest == "" { + if desc.Digest.IsZero() { // Returning a digest isn't mandatory according to the spec, and // at least one registry (AWS's ECR) fails to return a digest // when doing a GET of a tag. @@ -165,7 +168,7 @@ func (c *client) read(ctx context.Context, rreq *ocirequest.Request) (_ oci.Blob if int64(len(data)) != desc.Size { return nil, fmt.Errorf("body size mismatch") } - desc.Digest = digest.FromBytes(data) + desc.Digest = ocidigest.FromBytes(data) resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(data)) } else { @@ -176,7 +179,8 @@ func (c *client) read(ctx context.Context, rreq *ocirequest.Request) (_ oci.Blob return nil, err } resp1.Body.Close() - desc, err = descriptorFromResponse(resp1, oci.Digest(rreq1.Digest), requireSize|requireDigest) + knownDigest, _ := ocidigest.Parse(rreq1.Digest) + desc, err = descriptorFromResponse(resp1, knownDigest, requireSize|requireDigest) if err != nil { return nil, err } diff --git a/ociclient/referrers_test.go b/ociclient/referrers_test.go index 366a52d..da7721e 100644 --- a/ociclient/referrers_test.go +++ b/ociclient/referrers_test.go @@ -11,8 +11,7 @@ import ( "testing" "github.com/docker/oci" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/docker/oci/ocidigest" "github.com/stretchr/testify/require" "github.com/docker/oci/ociclient" @@ -39,28 +38,28 @@ func TestReferrersFallback(t *testing.T) { config := pushScratchConfig(t, client, repo) // Push a manifest to refer to. - subject := pushManifest(t, client, repo, "sometag", &oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, - Config: withMediaType(config, "subject/mediatype"), - }, ocispec.MediaTypeImageManifest) + subject := pushManifest(t, client, repo, "sometag", &oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, + Config: ref(withMediaType(config, "subject/mediatype")), + }, oci.MediaTypeImageManifest) - index := &ocispec.Index{ - MediaType: ocispec.MediaTypeImageIndex, + index := &oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageIndex, } // Then push some manifests that refer to it and update the index at the same time. for i := range 5 { artifactType := fmt.Sprintf("referrer/%d", i) - desc := pushManifest(t, client, repo, "", &oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, + desc := pushManifest(t, client, repo, "", &oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, Subject: &subject, - Config: withMediaType(config, artifactType), - }, ocispec.MediaTypeImageManifest) + Config: ref(withMediaType(config, artifactType)), + }, oci.MediaTypeImageManifest) desc.ArtifactType = artifactType index.Manifests = append(index.Manifests, desc) } // Then push the index to the referrers tag. - pushManifest(t, client, repo, strings.ReplaceAll(string(subject.Digest), ":", "-"), index, ocispec.MediaTypeImageIndex) + pushManifest(t, client, repo, strings.ReplaceAll(subject.Digest.String(), ":", "-"), index, oci.MediaTypeImageIndex) // Then ask for the referrers. var got []oci.Descriptor @@ -84,10 +83,14 @@ func withMediaType(desc oci.Descriptor, mediaType string) oci.Descriptor { return desc } +func ref[T any](x T) *T { + return &x +} + func pushScratchConfig(t *testing.T, client oci.Interface, repo string) oci.Descriptor { content := []byte("{}") - desc := ocispec.Descriptor{ - Digest: digest.FromBytes(content), + desc := oci.Descriptor{ + Digest: ocidigest.FromBytes(content), Size: int64(len(content)), } _, err := client.PushBlob(context.Background(), repo, desc, bytes.NewReader(content)) diff --git a/ociclient/referrerstag_test.go b/ociclient/referrerstag_test.go index 622aefb..53129d7 100644 --- a/ociclient/referrerstag_test.go +++ b/ociclient/referrerstag_test.go @@ -4,29 +4,34 @@ import ( "testing" "github.com/docker/oci" + "github.com/docker/oci/ocidigest" "github.com/stretchr/testify/require" ) +func mustDigest(s string) oci.Digest { + digest, err := ocidigest.Parse(s) + if err != nil { + panic(err) + } + return digest +} + var referrersTagTests = []struct { digest oci.Digest want string }{{ // Test case from the distribution spec. - digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + digest: mustDigest("sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), want: "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", }, { // Test case from the distribution spec. - digest: "sha512:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + digest: mustDigest("sha512:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), want: "sha512-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", -}, { - // Test case from the distribution spec. - digest: "test+algorithm+using+algorithm+separators+and+lots+of+characters+to+excercise+overall+truncation:alsoSome=InTheEncodedSectionToShowHyphenReplacementAndLotsAndLotsOfCharactersToExcerciseEncodedTruncation", - want: "test-algorithm-using-algorithm-s-alsoSome-InTheEncodedSectionToShowHyphenReplacementAndLotsAndLot", }} func TestReferrersTag(t *testing.T) { for _, test := range referrersTagTests { - t.Run(string(test.digest), func(t *testing.T) { + t.Run(test.digest.String(), func(t *testing.T) { require.Equal(t, test.want, referrersTag(test.digest)) }) } diff --git a/ociclient/writer.go b/ociclient/writer.go index a094862..d439761 100644 --- a/ociclient/writer.go +++ b/ociclient/writer.go @@ -26,10 +26,10 @@ import ( "sync" "github.com/docker/oci" - "github.com/opencontainers/go-digest" "github.com/docker/oci/internal/ocirequest" "github.com/docker/oci/ociauth" + "github.com/docker/oci/ocidigest" ) // This file implements the oci.Writer methods. @@ -39,10 +39,10 @@ func (c *client) PushManifest(ctx context.Context, repo string, contents []byte, return oci.Descriptor{}, fmt.Errorf("PushManifest called with empty mediaType") } var dig oci.Digest - if params != nil && params.Digest != "" { + if params != nil && !params.Digest.IsZero() { dig = params.Digest } else { - dig = digest.FromBytes(contents) + dig = ocidigest.FromBytes(contents) } desc := oci.Descriptor{ Digest: dig, @@ -62,7 +62,7 @@ func (c *client) PushManifest(ctx context.Context, repo string, contents []byte, rreq := &ocirequest.Request{ Kind: ocirequest.ReqManifestPut, Repo: repo, - Digest: string(desc.Digest), + Digest: desc.Digest.String(), } _, err := c.putManifest(ctx, rreq, desc) return desc, err @@ -71,7 +71,7 @@ func (c *client) PushManifest(ctx context.Context, repo string, contents []byte, Kind: ocirequest.ReqManifestPut, Repo: repo, Tags: tags, - Digest: string(desc.Digest), + Digest: desc.Digest.String(), } createdTags, err := c.putManifest(ctx, rreq, desc) if err != nil || len(createdTags) != len(tags) { @@ -81,7 +81,7 @@ func (c *client) PushManifest(ctx context.Context, repo string, contents []byte, Kind: ocirequest.ReqManifestPut, Repo: repo, Tag: tag, - Digest: string(desc.Digest), + Digest: desc.Digest.String(), } _, err = c.putManifest(ctx, rreq, desc) if err != nil { @@ -120,7 +120,7 @@ func (c *client) MountBlob(ctx context.Context, fromRepo, toRepo string, dig oci Kind: ocirequest.ReqBlobMount, Repo: toRepo, FromRepo: fromRepo, - Digest: string(dig), + Digest: dig.String(), } resp, err := c.doRequest(ctx, rreq, http.StatusCreated, http.StatusAccepted) if err != nil { @@ -171,7 +171,7 @@ func (c *client) PushBlob(ctx context.Context, repo string, desc oci.Descriptor, if err != nil { return oci.Descriptor{}, err } - req.URL = urlWithDigest(location, string(desc.Digest)) + req.URL = urlWithDigest(location, desc.Digest.String()) req.ContentLength = desc.Size req.Header.Set("Content-Type", "application/octet-stream") // TODO: per the spec, the content-range header here is unnecessary. @@ -332,7 +332,7 @@ func (w *blobWriter) Write(buf []byte) (int, error) { // then followed by an empty-bodied PUT with the call to Commit. // Instead, we want the writes to not flush at all, and Commit to PUT the entire chunk. if len(w.chunk)+len(buf) > w.chunkSize { - if err := w.flush(buf, ""); err != nil { + if err := w.flush(buf, oci.Digest{}); err != nil { return 0, err } } else { @@ -349,19 +349,19 @@ func (w *blobWriter) Write(buf []byte) (int, error) { // If commitDigest is non-empty, this is the final segment of data in the blob: // the blob is being committed and the digest should hold the digest of the entire blob content. func (w *blobWriter) flush(buf []byte, commitDigest oci.Digest) error { - if commitDigest == "" && len(buf)+len(w.chunk) == 0 { + if commitDigest.IsZero() && len(buf)+len(w.chunk) == 0 { return nil } // Start a new PATCH request to send the currently outstanding data. method := "PATCH" expect := http.StatusAccepted reqURL := w.location - if commitDigest != "" { + if !commitDigest.IsZero() { // This is the final piece of data, so send it as the final PUT request // (committing the whole blob) which avoids an extra round trip. method = "PUT" expect = http.StatusCreated - reqURL = urlWithDigest(reqURL, string(commitDigest)) + reqURL = urlWithDigest(reqURL, commitDigest.String()) } req, err := http.NewRequestWithContext(w.ctx, method, "", concatBody(w.chunk, buf)) if err != nil { @@ -410,7 +410,7 @@ func (w *blobWriter) Close() error { if w.closed { return w.closeErr } - err := w.flush(nil, "") + err := w.flush(nil, oci.Digest{}) w.closed = true w.closeErr = err return err @@ -433,7 +433,7 @@ func (w *blobWriter) ID() string { } func (w *blobWriter) Commit(digest oci.Digest) (oci.Descriptor, error) { - if digest == "" { + if digest.IsZero() { return oci.Descriptor{}, fmt.Errorf("cannot commit with an empty digest") } w.mu.Lock() diff --git a/ocidigest/README.md b/ocidigest/README.md new file mode 100644 index 0000000..8c5cab0 --- /dev/null +++ b/ocidigest/README.md @@ -0,0 +1,350 @@ +# ocidigest + +`ocidigest` calculates and verifies OCI-compatible content digests: + +```text +: +``` + +The built-in algorithms are `sha256` and `sha512`. `sha256` is the canonical +default. + +## Parse a Digest + +```go +dgst, err := ocidigest.Parse("sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9") +if err != nil { + return err +} + +fmt.Println(dgst.Algorithm()) // sha256 +fmt.Println(dgst.Encoded()) // b94d27... +fmt.Println(dgst.String()) // sha256:b94d27... +``` + +`Digest` values are comparable and can be used as map keys: + +```go +seen := map[ocidigest.Digest]bool{ + dgst: true, +} +``` + +## Digest Bytes + +Use `FromBytes` when content is already buffered, such as manifest JSON or test data: + +```go +payload := []byte(`{"schemaVersion":2}`) +dgst := ocidigest.FromBytes(payload) +``` + +To choose a non-canonical algorithm: + +```go +dgst := ocidigest.SHA512.FromBytes(payload) +``` + +## Digest a Reader + +Use `FromReader` when content should be consumed and discarded after hashing: + +```go +f, err := os.Open("layer.tar") +if err != nil { + return err +} +defer f.Close() + +dgst, err := ocidigest.FromReader(f) +if err != nil { + return err +} +``` + +For a specific algorithm: + +```go +dgst, err := ocidigest.SHA512.FromReader(f) +``` + +## Digest While Copying + +Create a `Digester` and write bytes through it directly: + +```go +d, err := ocidigest.SHA256.New() +if err != nil { + return err +} + +if _, err := io.Copy(d, src); err != nil { + return err +} + +dgst, err := d.Digest() +if err != nil { + return err +} +``` + +Use `io.TeeReader` when bytes should be copied somewhere else while hashing: + +```go +d, err := ocidigest.SHA256.New() +if err != nil { + return err +} + +tee := io.TeeReader(src, d) +if _, err := io.Copy(dst, tee); err != nil { + return err +} + +dgst, err := d.Digest() +``` + +## Digesting Reader Wrapper + +`NewReader` passes reads through to an underlying reader and records the digest: + +```go +r, err := ocidigest.NewReader(src, ocidigest.SHA256) +if err != nil { + return err +} + +if _, err := io.Copy(dst, r); err != nil { + return err +} + +dgst, err := r.Digest() +``` + +The wrapper also exposes how many bytes were digested: + +```go +fmt.Println(r.Size()) +``` + +## Digesting Writer Wrapper + +`NewWriter` passes writes through to an underlying writer and records the digest: + +```go +w, err := ocidigest.NewWriter(dst, ocidigest.SHA256) +if err != nil { + return err +} + +if _, err := io.Copy(w, src); err != nil { + return err +} + +dgst, err := w.Digest() +``` + +Pass `nil` as the underlying writer to only calculate the digest: + +```go +w, err := ocidigest.NewWriter(nil, ocidigest.SHA256) +if err != nil { + return err +} +_, err = w.Write(payload) +``` + +## Verify a Stream + +`NewVerifier` chooses the hashing algorithm from the expected digest: + +```go +expected, err := ocidigest.Parse("sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9") +if err != nil { + return err +} + +v, err := ocidigest.NewVerifier(expected) +if err != nil { + return err +} + +if _, err := io.Copy(v, src); err != nil { + return err +} +if !v.Verified() { + return ocidigest.ErrDigestMismatch +} +``` + +## Verify While Reading + +`VerifierReader` reports a digest mismatch as a read error at EOF: + +```go +vr, err := ocidigest.NewVerifierReader(src, expected) +if err != nil { + return err +} + +if _, err := io.Copy(dst, vr); err != nil { + if errors.Is(err, ocidigest.ErrDigestMismatch) { + return fmt.Errorf("content digest did not match: %w", err) + } + return err +} + +if !vr.Verified() { + return ocidigest.ErrDigestMismatch +} +``` + +If the final read returns bytes and EOF together, `VerifierReader` returns the final bytes +first. The EOF or mismatch is returned by the next read. This preserves normal `io.Reader` +byte-delivery behavior. + +## Resume Digest State + +Digest state can be exported and restored for short-lived resumable workflows, such as +chunked uploads. State is only intended for the same library version and same algorithm +implementation. + +```go +d, err := ocidigest.SHA256.New() +if err != nil { + return err +} + +if _, err := d.Write(firstChunk); err != nil { + return err +} + +state, err := d.State() +if err != nil { + return err +} + +// Persist state alongside the upload ID and offset. +save(state) +``` + +Later: + +```go +state := load() + +d, err := ocidigest.NewFromState(state) +if err != nil { + return err +} + +if d.Size() != expectedUploadOffset { + return fmt.Errorf("digest state offset mismatch") +} + +if _, err := d.Write(nextChunk); err != nil { + return err +} + +finalDigest, err := d.Digest() +``` + +Reset clears both the hash state and byte count: + +```go +d.Reset() +fmt.Println(d.Size()) // 0 +``` + +## Multiple Digests from One Stream + +Use `MultiDigester` when one pass should produce several digests: + +```go +m, err := ocidigest.NewMultiDigester(ocidigest.SHA256, ocidigest.SHA512) +if err != nil { + return err +} + +if _, err := io.Copy(m, src); err != nil { + return err +} + +digests, err := m.Digests() +if err != nil { + return err +} + +sha256Digest := digests[0] +sha512Digest := digests[1] +``` + +Or wrap a reader: + +```go +r, err := ocidigest.NewMultiReader(src, ocidigest.SHA256, ocidigest.SHA512) +if err != nil { + return err +} + +if _, err := io.Copy(dst, r); err != nil { + return err +} + +digests, err := r.Digests() +``` + +Multi-digest state can also be exported and restored: + +```go +state, err := m.States() +if err != nil { + return err +} + +restored, err := ocidigest.NewMultiDigesterFromState(state) +if err != nil { + return err +} +``` + +## Register a Custom Algorithm + +Algorithms are registered globally by name: + +```go +alg, err := ocidigest.RegisterAlgorithm(ocidigest.RegisterOptions{ + Name: "mysha256", + Size: sha256.Size, + Encoding: ocidigest.EncodeHex{Len: sha256.Size * 2}, + NewHash: sha256.New, +}) +if err != nil { + return err +} + +dgst := alg.FromBytes(payload) +fmt.Println(dgst.String()) // mysha256: +``` + +Custom algorithms that need resumable state can provide `MarshalState` and `NewState`. +Most callers should not need this unless the hash implementation does not support Go's +standard binary marshaling interfaces. + +## JSON and Text Encoding + +`Digest` marshals as a string: + +```go +data, err := json.Marshal(dgst) +// "sha256:b94d27..." +``` + +And unmarshals through the same validation as `Parse`: + +```go +var dgst ocidigest.Digest +if err := json.Unmarshal(data, &dgst); err != nil { + return err +} +``` diff --git a/ocidigest/algorithm.go b/ocidigest/algorithm.go new file mode 100644 index 0000000..cab710a --- /dev/null +++ b/ocidigest/algorithm.go @@ -0,0 +1,199 @@ +package ocidigest + +import ( + "crypto/sha256" + "crypto/sha512" + "fmt" + "hash" + "io" + "regexp" + "sync" +) + +// Algorithm identifies a digest algorithm and its encoding. +type Algorithm struct { + name string +} + +// RegisterOptions defines a digest algorithm implementation. +type RegisterOptions struct { + Name string + Size int + Encoding Encoding + NewHash func() hash.Hash + MarshalState func(hash.Hash) (encoding string, payload []byte, err error) + NewState func(State) (hash.Hash, error) +} + +type algorithmInfo struct { + name string + size int + encoding Encoding + newHash func() hash.Hash + marshalState func(hash.Hash) (encoding string, payload []byte, err error) + newState func(State) (hash.Hash, error) +} + +var ( + SHA256 Algorithm + SHA512 Algorithm + Canonical Algorithm + + algorithmsMu sync.RWMutex + algorithms = map[string]algorithmInfo{} + + algorithmNameRegexp = regexp.MustCompile(`^[a-z0-9]+([+._-][a-z0-9]+)*$`) +) + +func init() { + var err error + SHA256, err = RegisterAlgorithm(RegisterOptions{ + Name: "sha256", + Size: sha256.Size, + Encoding: EncodeHex{Len: sha256.Size * 2}, + NewHash: sha256.New, + }) + if err != nil { + panic(err) + } + SHA512, err = RegisterAlgorithm(RegisterOptions{ + Name: "sha512", + Size: sha512.Size, + Encoding: EncodeHex{Len: sha512.Size * 2}, + NewHash: sha512.New, + }) + if err != nil { + panic(err) + } + Canonical = SHA256 +} + +// LookupAlgorithm returns a previously registered algorithm by name. +func LookupAlgorithm(name string) (Algorithm, error) { + info, err := lookupAlgorithm(name) + if err != nil { + return Algorithm{}, err + } + return Algorithm{name: info.name}, nil +} + +// RegisterAlgorithm registers a digest algorithm. +func RegisterAlgorithm(opts RegisterOptions) (Algorithm, error) { + if !algorithmNameRegexp.MatchString(opts.Name) { + return Algorithm{}, fmt.Errorf("%w: %q", ErrAlgorithmInvalidName, opts.Name) + } + if algorithmRegistered(opts.Name) { + return Algorithm{}, fmt.Errorf("%w: %q", ErrAlgorithmExists, opts.Name) + } + if opts.NewHash == nil { + return Algorithm{}, fmt.Errorf("%w: %q", ErrHashInvalid, opts.Name) + } + h := opts.NewHash() + if h == nil { + return Algorithm{}, fmt.Errorf("%w: %q", ErrHashInvalid, opts.Name) + } + if h.Size() <= 0 { + return Algorithm{}, fmt.Errorf("%w: %q", ErrHashInvalid, opts.Name) + } + if n, err := h.Write([]byte("ocidigest")); err != nil || n != len("ocidigest") { + return Algorithm{}, fmt.Errorf("%w: %q write failed", ErrHashInvalid, opts.Name) + } + h.Reset() + size := opts.Size + if size == 0 { + size = h.Size() + } + if h.Size() != size { + return Algorithm{}, fmt.Errorf("%w: %q has hash size %d, want %d", ErrHashInvalid, opts.Name, h.Size(), size) + } + if err := validateEncoding(opts.Encoding, size); err != nil { + return Algorithm{}, err + } + + algorithmsMu.Lock() + defer algorithmsMu.Unlock() + if _, ok := algorithms[opts.Name]; ok { + return Algorithm{}, fmt.Errorf("%w: %q", ErrAlgorithmExists, opts.Name) + } + algorithms[opts.Name] = algorithmInfo{ + name: opts.Name, + size: size, + encoding: opts.Encoding, + newHash: opts.NewHash, + marshalState: opts.MarshalState, + newState: opts.NewState, + } + return Algorithm{name: opts.Name}, nil +} + +func algorithmRegistered(name string) bool { + algorithmsMu.RLock() + defer algorithmsMu.RUnlock() + _, ok := algorithms[name] + return ok +} + +// String returns the algorithm name. +func (a Algorithm) String() string { + return a.name +} + +// Size returns the digest size in bytes. +func (a Algorithm) Size() int { + info, err := lookupAlgorithm(a.name) + if err != nil { + return 0 + } + return info.size +} + +// New returns a new digester for the algorithm. +func (a Algorithm) New() (Digester, error) { + info, err := lookupAlgorithm(a.name) + if err != nil { + return nil, err + } + return newDigester(info, 0), nil +} + +// FromBytes returns the digest of p. +func (a Algorithm) FromBytes(p []byte) Digest { + d, err := a.New() + if err != nil { + return Digest{} + } + _, _ = d.Write(p) + digest, err := d.Digest() + if err != nil { + return Digest{} + } + return digest +} + +// FromReader reads r to EOF and returns its digest. +func (a Algorithm) FromReader(r io.Reader) (Digest, error) { + d, err := a.New() + if err != nil { + return Digest{}, err + } + if _, err := io.Copy(d, r); err != nil { + return Digest{}, err + } + return d.Digest() +} + +func lookupAlgorithm(name string) (algorithmInfo, error) { + if name == "" { + return algorithmInfo{}, ErrAlgorithmInvalidName + } + if !algorithmNameRegexp.MatchString(name) { + return algorithmInfo{}, fmt.Errorf("%w: %q", ErrAlgorithmInvalidName, name) + } + algorithmsMu.RLock() + defer algorithmsMu.RUnlock() + info, ok := algorithms[name] + if !ok { + return algorithmInfo{}, fmt.Errorf("%w: %q", ErrAlgorithmUnknown, name) + } + return info, nil +} diff --git a/ocidigest/digest.go b/ocidigest/digest.go new file mode 100644 index 0000000..c5280d2 --- /dev/null +++ b/ocidigest/digest.go @@ -0,0 +1,147 @@ +package ocidigest + +import ( + "encoding/json" + "fmt" + "hash" + "io" + "strings" +) + +// Digest is an OCI-compatible digest in the form ":". +type Digest struct { + alg string + enc string +} + +// Parse parses and validates a digest string. +func Parse(s string) (Digest, error) { + if s == "" { + return Digest{}, nil + } + alg, enc, ok := strings.Cut(s, ":") + if !ok || alg == "" || enc == "" { + return Digest{}, fmt.Errorf("%w: %q", ErrDigestInvalid, s) + } + return NewDigestFromEncoded(Algorithm{name: alg}, enc) +} + +// FromBytes returns the canonical digest of p. +func FromBytes(p []byte) Digest { + return Canonical.FromBytes(p) +} + +// FromReader reads r to EOF and returns its canonical digest. +func FromReader(r io.Reader) (Digest, error) { + return Canonical.FromReader(r) +} + +// NewDigest returns a digest from alg and the current state of h. +func NewDigest(alg Algorithm, h hash.Hash) (Digest, error) { + if h == nil { + return Digest{}, ErrHashInvalid + } + info, err := lookupAlgorithm(alg.name) + if err != nil { + return Digest{}, err + } + if h.Size() != info.size { + return Digest{}, fmt.Errorf("%w: %q has hash size %d, want %d", ErrHashInvalid, alg.name, h.Size(), info.size) + } + enc, err := info.encoding.Encode(h.Sum(nil)) + if err != nil { + return Digest{}, err + } + return Digest{alg: info.name, enc: enc}, nil +} + +// NewDigestFromEncoded returns a digest from alg and an already encoded hash value. +func NewDigestFromEncoded(alg Algorithm, encoded string) (Digest, error) { + info, err := lookupAlgorithm(alg.name) + if err != nil { + return Digest{}, err + } + if !info.encoding.Validate(encoded) { + return Digest{}, fmt.Errorf("%w: %q", ErrEncodingInvalid, encoded) + } + return Digest{alg: info.name, enc: encoded}, nil +} + +// Algorithm returns the digest algorithm. +func (d Digest) Algorithm() Algorithm { + return Algorithm{name: d.alg} +} + +// Encoded returns the encoded hash portion of the digest. +func (d Digest) Encoded() string { + return d.enc +} + +// String returns the string form of the digest. +func (d Digest) String() string { + if d.IsZero() { + return "" + } + if d.alg == "" || d.enc == "" { + return "" + } + return d.alg + ":" + d.enc +} + +// IsZero reports whether d is the zero digest. +func (d Digest) IsZero() bool { + return d.alg == "" && d.enc == "" +} + +// Equal reports whether d and cmp are identical. +func (d Digest) Equal(cmp Digest) bool { + return d == cmp +} + +// Validate checks whether d is a valid non-zero digest. +func (d Digest) Validate() error { + if d.IsZero() { + return ErrDigestInvalid + } + info, err := lookupAlgorithm(d.alg) + if err != nil { + return err + } + if !info.encoding.Validate(d.enc) { + return fmt.Errorf("%w: %q", ErrEncodingInvalid, d.enc) + } + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (d Digest) MarshalText() ([]byte, error) { + return []byte(d.String()), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (d *Digest) UnmarshalText(text []byte) error { + parsed, err := Parse(string(text)) + if err != nil { + return err + } + *d = parsed + return nil +} + +// MarshalJSON encodes d as a JSON string. +func (d Digest) MarshalJSON() ([]byte, error) { + return json.Marshal(d.String()) +} + +// UnmarshalJSON decodes d from a JSON string. +func (d *Digest) UnmarshalJSON(data []byte) error { + if string(data) == "null" { + *d = Digest{} + return nil + } + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + return d.UnmarshalText([]byte(s)) +} diff --git a/ocidigest/digest_test.go b/ocidigest/digest_test.go new file mode 100644 index 0000000..bfe6b4e --- /dev/null +++ b/ocidigest/digest_test.go @@ -0,0 +1,417 @@ +package ocidigest + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "hash" + "io" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + helloWorldSHA256 = "sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9" + helloWorldSHA512 = "sha512:309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f" +) + +func TestDigestParseValidateAndComparable(t *testing.T) { + d, err := Parse(helloWorldSHA256) + require.NoError(t, err) + + require.Equal(t, "sha256", d.Algorithm().String()) + require.Equal(t, strings.TrimPrefix(helloWorldSHA256, "sha256:"), d.Encoded()) + require.Equal(t, helloWorldSHA256, d.String()) + require.NoError(t, d.Validate()) + + byDigest := map[Digest]string{d: "hello"} + require.Equal(t, "hello", byDigest[d]) + require.True(t, d.Equal(d)) + + _, err = Parse("sha256:B94D27B9934D3E08A52E52D7DA7DABFADEB100F1FDDC5EF7A679E9C58EE68F") + require.ErrorIs(t, err, ErrEncodingInvalid) + + _, err = Parse("not-a-digest") + require.ErrorIs(t, err, ErrDigestInvalid) + + _, err = Parse("bad/alg:abcdef") + require.ErrorIs(t, err, ErrAlgorithmInvalidName) + + var zero Digest + require.True(t, zero.IsZero()) + require.Equal(t, "", zero.String()) + require.ErrorIs(t, zero.Validate(), ErrDigestInvalid) +} + +func TestDigestJSONAndText(t *testing.T) { + d := FromBytes([]byte("hello world")) + + text, err := d.MarshalText() + require.NoError(t, err) + require.Equal(t, helloWorldSHA256, string(text)) + + var fromText Digest + require.NoError(t, fromText.UnmarshalText(text)) + require.Equal(t, d, fromText) + + data, err := json.Marshal(d) + require.NoError(t, err) + require.JSONEq(t, `"`+helloWorldSHA256+`"`, string(data)) + + var fromJSON Digest + require.NoError(t, json.Unmarshal(data, &fromJSON)) + require.Equal(t, d, fromJSON) + + var zero Digest + data, err = json.Marshal(zero) + require.NoError(t, err) + require.JSONEq(t, `""`, string(data)) + + fromJSON = Digest{alg: "sha256", enc: "bad"} + require.NoError(t, json.Unmarshal([]byte("null"), &fromJSON)) + require.True(t, fromJSON.IsZero()) +} + +func TestDigestCreationAPIs(t *testing.T) { + data := []byte("hello world") + + require.Equal(t, helloWorldSHA256, FromBytes(data).String()) + require.Equal(t, helloWorldSHA256, SHA256.FromBytes(data).String()) + require.Equal(t, helloWorldSHA512, SHA512.FromBytes(data).String()) + + fromReader, err := FromReader(bytes.NewReader(data)) + require.NoError(t, err) + require.Equal(t, FromBytes(data), fromReader) + + sha512FromReader, err := SHA512.FromReader(strings.NewReader("hello world")) + require.NoError(t, err) + require.Equal(t, helloWorldSHA512, sha512FromReader.String()) + + dr, err := NewReader(bytes.NewReader(data), SHA256) + require.NoError(t, err) + read, err := io.ReadAll(dr) + require.NoError(t, err) + require.Equal(t, data, read) + got, err := dr.Digest() + require.NoError(t, err) + require.Equal(t, FromBytes(data), got) + require.Equal(t, int64(len(data)), dr.Size()) + + var buf bytes.Buffer + w, err := NewWriter(&buf, SHA256) + require.NoError(t, err) + n, err := w.Write(data) + require.NoError(t, err) + require.Equal(t, len(data), n) + require.Equal(t, string(data), buf.String()) + got, err = w.Digest() + require.NoError(t, err) + require.Equal(t, FromBytes(data), got) + require.Equal(t, int64(len(data)), w.Size()) +} + +func TestVerifier(t *testing.T) { + expected := FromBytes([]byte("hello world")) + v, err := NewVerifier(expected) + require.NoError(t, err) + require.False(t, v.Verified()) + + n, err := v.Write([]byte("hello world")) + require.NoError(t, err) + require.Equal(t, len("hello world"), n) + require.True(t, v.Verified()) + + got, err := v.Calculated() + require.NoError(t, err) + require.Equal(t, expected, got) + require.Equal(t, expected, v.Digest()) +} + +func TestVerifierReaderMismatchOnEOF(t *testing.T) { + expected := FromBytes([]byte("hello world")) + vr, err := NewVerifierReader(strings.NewReader("goodbye"), expected) + require.NoError(t, err) + + _, err = io.Copy(io.Discard, vr) + require.ErrorIs(t, err, ErrDigestMismatch) + require.False(t, vr.Verified()) + + got, calcErr := vr.Calculated() + require.NoError(t, calcErr) + require.NotEqual(t, expected, got) +} + +func TestVerifierReaderReturnsFinalBytesBeforeFinalVerification(t *testing.T) { + expected := FromBytes([]byte("abc")) + vr, err := NewVerifierReader(&eofWithBytesReader{data: []byte("abc")}, expected) + require.NoError(t, err) + + buf := make([]byte, 8) + n, err := vr.Read(buf) + require.NoError(t, err) + require.Equal(t, 3, n) + require.Equal(t, "abc", string(buf[:n])) + require.False(t, vr.Verified()) + + n, err = vr.Read(buf) + require.Equal(t, 0, n) + require.ErrorIs(t, err, io.EOF) + require.True(t, vr.Verified()) +} + +func TestStateRoundTripResetAndOffset(t *testing.T) { + d, err := SHA256.New() + require.NoError(t, err) + n, err := d.Write([]byte("hello ")) + require.NoError(t, err) + require.Equal(t, len("hello "), n) + + st, err := d.State() + require.NoError(t, err) + require.Equal(t, "sha256", st.Algorithm) + require.Equal(t, int64(len("hello ")), st.Offset) + require.Equal(t, d.Size(), st.Offset) + require.NotEmpty(t, st.Payload) + + resumed, err := NewFromState(st) + require.NoError(t, err) + require.Equal(t, st.Offset, resumed.Size()) + _, err = resumed.Write([]byte("world")) + require.NoError(t, err) + + got, err := resumed.Digest() + require.NoError(t, err) + require.Equal(t, FromBytes([]byte("hello world")), got) + + resumed.Reset() + require.Equal(t, int64(0), resumed.Size()) + _, err = resumed.Write([]byte("hello world")) + require.NoError(t, err) + got, err = resumed.Digest() + require.NoError(t, err) + require.Equal(t, FromBytes([]byte("hello world")), got) +} + +func TestMultiDigester(t *testing.T) { + m, err := NewMultiDigester(SHA256, SHA512) + require.NoError(t, err) + + _, err = m.Write([]byte("hello world")) + require.NoError(t, err) + require.Equal(t, int64(len("hello world")), m.Size()) + + digests, err := m.Digests() + require.NoError(t, err) + require.Equal(t, []Digest{ + SHA256.FromBytes([]byte("hello world")), + SHA512.FromBytes([]byte("hello world")), + }, digests) + + sha512Digest, err := m.Digest(SHA512) + require.NoError(t, err) + require.Equal(t, helloWorldSHA512, sha512Digest.String()) + + _, err = NewMultiDigester(SHA256, SHA256) + require.ErrorIs(t, err, ErrAlgorithmDuplicate) + + m.Reset() + require.Equal(t, int64(0), m.Size()) +} + +func TestMultiStateRoundTrip(t *testing.T) { + m, err := NewMultiDigester(SHA256, SHA512) + require.NoError(t, err) + _, err = m.Write([]byte("hello ")) + require.NoError(t, err) + + st, err := m.States() + require.NoError(t, err) + require.Len(t, st.States, 2) + require.Equal(t, int64(len("hello ")), st.States[0].Offset) + require.Equal(t, st.States[0].Offset, st.States[1].Offset) + + resumed, err := NewMultiDigesterFromState(st) + require.NoError(t, err) + require.Equal(t, int64(len("hello ")), resumed.Size()) + _, err = resumed.Write([]byte("world")) + require.NoError(t, err) + + digests, err := resumed.Digests() + require.NoError(t, err) + require.Equal(t, []Digest{ + SHA256.FromBytes([]byte("hello world")), + SHA512.FromBytes([]byte("hello world")), + }, digests) +} + +func TestMultiReader(t *testing.T) { + r, err := NewMultiReader(strings.NewReader("hello world"), SHA256, SHA512) + require.NoError(t, err) + _, err = io.Copy(io.Discard, r) + require.NoError(t, err) + + digests, err := r.Digests() + require.NoError(t, err) + require.Equal(t, helloWorldSHA256, digests[0].String()) + require.Equal(t, helloWorldSHA512, digests[1].String()) +} + +func TestAlgorithmRegistrationErrors(t *testing.T) { + _, err := RegisterAlgorithm(RegisterOptions{ + Name: "sha256", + Size: 32, + Encoding: EncodeHex{Len: 64}, + NewHash: nil, + }) + require.ErrorIs(t, err, ErrAlgorithmExists) + + _, err = RegisterAlgorithm(RegisterOptions{ + Name: "testnilhash", + Size: 32, + Encoding: EncodeHex{Len: 64}, + NewHash: nil, + }) + require.ErrorIs(t, err, ErrHashInvalid) + + _, err = RegisterAlgorithm(RegisterOptions{ + Name: "sha256", + Size: 32, + Encoding: EncodeHex{Len: 64}, + NewHash: sha256.New, + }) + require.ErrorIs(t, err, ErrAlgorithmExists) + + _, err = RegisterAlgorithm(RegisterOptions{ + Name: "bad/alg", + Size: 32, + Encoding: EncodeHex{Len: 64}, + NewHash: sha256.New, + }) + require.ErrorIs(t, err, ErrAlgorithmInvalidName) + + _, err = LookupAlgorithm("missing") + require.ErrorIs(t, err, ErrAlgorithmUnknown) +} + +func TestRegisterAlgorithmExtensibility(t *testing.T) { + alg, err := RegisterAlgorithm(RegisterOptions{ + Name: "testsha256", + Size: sha256.Size, + Encoding: EncodeHex{Len: sha256.Size * 2}, + NewHash: sha256.New, + }) + require.NoError(t, err) + + d := alg.FromBytes([]byte("hello world")) + require.Equal(t, "testsha256:"+strings.TrimPrefix(helloWorldSHA256, "sha256:"), d.String()) + + parsed, err := Parse(d.String()) + require.NoError(t, err) + require.Equal(t, d, parsed) +} + +func TestRegisterAlgorithmCustomState(t *testing.T) { + alg, err := RegisterAlgorithm(RegisterOptions{ + Name: "teststatehash", + Size: sha256.Size, + Encoding: EncodeHex{Len: sha256.Size * 2}, + NewHash: func() hash.Hash { + return &bufferHash{} + }, + MarshalState: func(h hash.Hash) (string, []byte, error) { + bh, ok := h.(*bufferHash) + if !ok { + return "", nil, ErrHashInvalid + } + return "buffer-v1", append([]byte(nil), bh.buf...), nil + }, + NewState: func(st State) (hash.Hash, error) { + if st.Encoding != "buffer-v1" { + return nil, ErrStateInvalid + } + return &bufferHash{buf: append([]byte(nil), st.Payload...)}, nil + }, + }) + require.NoError(t, err) + + d, err := alg.New() + require.NoError(t, err) + _, err = d.Write([]byte("hello ")) + require.NoError(t, err) + st, err := d.State() + require.NoError(t, err) + require.Equal(t, "buffer-v1", st.Encoding) + require.Equal(t, int64(len("hello ")), st.Offset) + + resumed, err := NewFromState(st) + require.NoError(t, err) + _, err = resumed.Write([]byte("world")) + require.NoError(t, err) + got, err := resumed.Digest() + require.NoError(t, err) + require.Equal(t, alg.FromBytes([]byte("hello world")), got) +} + +func TestInvalidState(t *testing.T) { + _, err := NewFromState(State{ + Algorithm: "sha256", + Encoding: "unknown", + }) + require.ErrorIs(t, err, ErrStateUnsupported) + + _, err = NewFromState(State{ + Algorithm: "sha256", + Encoding: "binary-marshaler-v1", + Offset: -1, + }) + require.ErrorIs(t, err, ErrStateInvalid) + + _, err = NewFromState(State{ + Algorithm: "sha256", + Encoding: "binary-marshaler-v1", + Payload: []byte("not a valid sha256 state"), + }) + require.ErrorIs(t, err, ErrStateInvalid) +} + +type eofWithBytesReader struct { + data []byte + done bool +} + +type bufferHash struct { + buf []byte +} + +func (h *bufferHash) Write(p []byte) (int, error) { + h.buf = append(h.buf, p...) + return len(p), nil +} + +func (h *bufferHash) Sum(b []byte) []byte { + sum := sha256.Sum256(h.buf) + return append(b, sum[:]...) +} + +func (h *bufferHash) Reset() { + h.buf = nil +} + +func (h *bufferHash) Size() int { + return sha256.Size +} + +func (h *bufferHash) BlockSize() int { + return sha256.BlockSize +} + +func (r *eofWithBytesReader) Read(p []byte) (int, error) { + if r.done { + return 0, io.EOF + } + copy(p, r.data) + r.done = true + return len(r.data), io.EOF +} diff --git a/ocidigest/digester.go b/ocidigest/digester.go new file mode 100644 index 0000000..de02572 --- /dev/null +++ b/ocidigest/digester.go @@ -0,0 +1,84 @@ +package ocidigest + +import ( + "hash" + "io" +) + +// Digester calculates a digest incrementally. +type Digester interface { + io.Writer + Algorithm() Algorithm + Digest() (Digest, error) + Reset() + Size() int64 + State() (State, error) +} + +type digester struct { + info algorithmInfo + hash hash.Hash + size int64 +} + +func newDigester(info algorithmInfo, size int64) *digester { + return &digester{ + info: info, + hash: info.newHash(), + size: size, + } +} + +func (d *digester) Write(p []byte) (int, error) { + if d == nil || d.hash == nil { + return 0, ErrHashInvalid + } + n, err := d.hash.Write(p) + d.size += int64(n) + return n, err +} + +func (d *digester) Algorithm() Algorithm { + if d == nil { + return Algorithm{} + } + return Algorithm{name: d.info.name} +} + +func (d *digester) Digest() (Digest, error) { + if d == nil || d.hash == nil { + return Digest{}, ErrHashInvalid + } + return NewDigest(Algorithm{name: d.info.name}, d.hash) +} + +func (d *digester) Reset() { + if d == nil || d.hash == nil { + return + } + d.hash.Reset() + d.size = 0 +} + +func (d *digester) Size() int64 { + if d == nil { + return 0 + } + return d.size +} + +func (d *digester) State() (State, error) { + if d == nil || d.hash == nil { + return State{}, ErrHashInvalid + } + encoding, payload, err := marshalHashState(d.info, d.hash) + if err != nil { + return State{}, err + } + return State{ + Algorithm: d.info.name, + Encoding: encoding, + Offset: d.size, + Payload: payload, + }, nil +} diff --git a/ocidigest/doc.go b/ocidigest/doc.go new file mode 100644 index 0000000..bf53580 --- /dev/null +++ b/ocidigest/doc.go @@ -0,0 +1,2 @@ +// Package ocidigest calculates and verifies OCI-compatible content digests. +package ocidigest diff --git a/ocidigest/encoding.go b/ocidigest/encoding.go new file mode 100644 index 0000000..e08c928 --- /dev/null +++ b/ocidigest/encoding.go @@ -0,0 +1,58 @@ +package ocidigest + +import ( + "encoding/hex" + "fmt" +) + +// Encoding encodes and validates the encoded hash portion of a digest. +type Encoding interface { + Encode([]byte) (string, error) + Validate(string) bool +} + +// EncodeHex encodes digest bytes as lowercase hexadecimal text. +type EncodeHex struct { + Len int +} + +// Encode returns p encoded as lowercase hexadecimal text. +func (e EncodeHex) Encode(p []byte) (string, error) { + if len(p)*2 != e.Len { + return "", ErrEncodingInvalid + } + return hex.EncodeToString(p), nil +} + +// Validate reports whether s is lowercase hexadecimal with the configured length. +func (e EncodeHex) Validate(s string) bool { + if len(s) != e.Len { + return false + } + for i := range len(s) { + c := s[i] + if ('0' <= c && c <= '9') || ('a' <= c && c <= 'f') { + continue + } + return false + } + return true +} + +func validateEncoding(enc Encoding, size int) error { + if enc == nil { + return fmt.Errorf("%w: nil encoding", ErrEncodingInvalid) + } + if size <= 0 { + return fmt.Errorf("%w: invalid size %d", ErrHashInvalid, size) + } + zero := make([]byte, size) + encoded, err := enc.Encode(zero) + if err != nil { + return fmt.Errorf("%w: %v", ErrEncodingInvalid, err) + } + if !enc.Validate(encoded) { + return fmt.Errorf("%w: encoded test value did not validate", ErrEncodingInvalid) + } + return nil +} diff --git a/ocidigest/errors.go b/ocidigest/errors.go new file mode 100644 index 0000000..5936079 --- /dev/null +++ b/ocidigest/errors.go @@ -0,0 +1,18 @@ +package ocidigest + +import "errors" + +var ( + ErrAlgorithmExists = errors.New("ocidigest: algorithm is already registered") + ErrAlgorithmInvalidName = errors.New("ocidigest: invalid algorithm name") + ErrAlgorithmUnknown = errors.New("ocidigest: algorithm is not registered") + ErrAlgorithmDuplicate = errors.New("ocidigest: duplicate algorithm") + ErrDigestInvalid = errors.New("ocidigest: digest is invalid") + ErrDigestMismatch = errors.New("ocidigest: digest mismatch") + ErrEncodingInvalid = errors.New("ocidigest: encoded digest is invalid") + ErrHashInvalid = errors.New("ocidigest: invalid hash") + ErrReaderInvalid = errors.New("ocidigest: invalid reader") + ErrStateInvalid = errors.New("ocidigest: state is invalid") + ErrStateUnsupported = errors.New("ocidigest: state is not supported") + ErrWriterInvalid = errors.New("ocidigest: invalid writer") +) diff --git a/ocidigest/multidigester.go b/ocidigest/multidigester.go new file mode 100644 index 0000000..85cd540 --- /dev/null +++ b/ocidigest/multidigester.go @@ -0,0 +1,218 @@ +package ocidigest + +import ( + "fmt" + "io" +) + +// MultiState is a serialized state for a MultiDigester. +type MultiState struct { + States []State `json:"states"` +} + +// MultiDigester calculates several digests over the same byte stream. +type MultiDigester struct { + order []Algorithm + digesters map[Algorithm]Digester +} + +// NewMultiDigester returns a digester for all provided algorithms. +func NewMultiDigester(algs ...Algorithm) (*MultiDigester, error) { + if len(algs) == 0 { + return nil, ErrAlgorithmInvalidName + } + m := &MultiDigester{ + order: make([]Algorithm, 0, len(algs)), + digesters: make(map[Algorithm]Digester, len(algs)), + } + for _, alg := range algs { + if alg.String() == "" { + return nil, ErrAlgorithmInvalidName + } + if _, ok := m.digesters[alg]; ok { + return nil, fmt.Errorf("%w: %q", ErrAlgorithmDuplicate, alg) + } + d, err := alg.New() + if err != nil { + return nil, err + } + m.order = append(m.order, alg) + m.digesters[alg] = d + } + return m, nil +} + +// NewMultiDigesterFromState restores a multi-digester from st. +func NewMultiDigesterFromState(st MultiState) (*MultiDigester, error) { + if len(st.States) == 0 { + return nil, ErrStateInvalid + } + m := &MultiDigester{ + order: make([]Algorithm, 0, len(st.States)), + digesters: make(map[Algorithm]Digester, len(st.States)), + } + var offset int64 = -1 + for _, state := range st.States { + d, err := NewFromState(state) + if err != nil { + return nil, err + } + alg := d.Algorithm() + if _, ok := m.digesters[alg]; ok { + return nil, fmt.Errorf("%w: %q", ErrAlgorithmDuplicate, alg) + } + if offset == -1 { + offset = d.Size() + } else if d.Size() != offset { + return nil, fmt.Errorf("%w: mismatched multi-state offsets", ErrStateInvalid) + } + m.order = append(m.order, alg) + m.digesters[alg] = d + } + return m, nil +} + +// Write writes p to every child digester. +func (m *MultiDigester) Write(p []byte) (int, error) { + if m == nil || len(m.order) == 0 { + return 0, ErrHashInvalid + } + for _, alg := range m.order { + if n, err := m.digesters[alg].Write(p); err != nil { + return n, err + } else if n != len(p) { + return n, io.ErrShortWrite + } + } + return len(p), nil +} + +// Algorithms returns the algorithms in digest output order. +func (m *MultiDigester) Algorithms() []Algorithm { + if m == nil { + return nil + } + return append([]Algorithm(nil), m.order...) +} + +// Digest returns the current digest for alg. +func (m *MultiDigester) Digest(alg Algorithm) (Digest, error) { + if m == nil { + return Digest{}, ErrHashInvalid + } + d, ok := m.digesters[alg] + if !ok { + return Digest{}, fmt.Errorf("%w: %q", ErrAlgorithmUnknown, alg) + } + return d.Digest() +} + +// Digests returns all current digests in algorithm order. +func (m *MultiDigester) Digests() ([]Digest, error) { + if m == nil { + return nil, ErrHashInvalid + } + digests := make([]Digest, 0, len(m.order)) + for _, alg := range m.order { + digest, err := m.digesters[alg].Digest() + if err != nil { + return nil, err + } + digests = append(digests, digest) + } + return digests, nil +} + +// Reset resets all child digesters. +func (m *MultiDigester) Reset() { + if m == nil { + return + } + for _, alg := range m.order { + m.digesters[alg].Reset() + } +} + +// Size returns the number of bytes written to each child digester. +func (m *MultiDigester) Size() int64 { + if m == nil || len(m.order) == 0 { + return 0 + } + return m.digesters[m.order[0]].Size() +} + +// State returns the state for alg. +func (m *MultiDigester) State(alg Algorithm) (State, error) { + if m == nil { + return State{}, ErrHashInvalid + } + d, ok := m.digesters[alg] + if !ok { + return State{}, fmt.Errorf("%w: %q", ErrAlgorithmUnknown, alg) + } + return d.State() +} + +// States returns all child states in algorithm order. +func (m *MultiDigester) States() (MultiState, error) { + if m == nil { + return MultiState{}, ErrHashInvalid + } + states := make([]State, 0, len(m.order)) + for _, alg := range m.order { + st, err := m.digesters[alg].State() + if err != nil { + return MultiState{}, err + } + states = append(states, st) + } + return MultiState{States: states}, nil +} + +// MultiReader wraps an io.Reader and records several digests. +type MultiReader struct { + r io.Reader + m *MultiDigester +} + +// NewMultiReader returns a multi-digesting reader for r. +func NewMultiReader(r io.Reader, algs ...Algorithm) (*MultiReader, error) { + if r == nil { + return nil, ErrReaderInvalid + } + m, err := NewMultiDigester(algs...) + if err != nil { + return nil, err + } + return &MultiReader{r: r, m: m}, nil +} + +// Read reads from the underlying reader and updates all digests. +func (r *MultiReader) Read(p []byte) (int, error) { + if r == nil || r.r == nil || r.m == nil { + return 0, ErrReaderInvalid + } + n, err := r.r.Read(p) + if n > 0 { + if _, writeErr := r.m.Write(p[:n]); writeErr != nil { + return n, writeErr + } + } + return n, err +} + +// Digests returns all current digests in algorithm order. +func (r *MultiReader) Digests() ([]Digest, error) { + if r == nil || r.m == nil { + return nil, ErrReaderInvalid + } + return r.m.Digests() +} + +// States returns all current digest states in algorithm order. +func (r *MultiReader) States() (MultiState, error) { + if r == nil || r.m == nil { + return MultiState{}, ErrReaderInvalid + } + return r.m.States() +} diff --git a/ocidigest/reader.go b/ocidigest/reader.go new file mode 100644 index 0000000..d16d9ba --- /dev/null +++ b/ocidigest/reader.go @@ -0,0 +1,62 @@ +package ocidigest + +import ( + "errors" + "io" +) + +// Reader wraps an io.Reader and records the digest of bytes read through it. +type Reader struct { + r io.Reader + d Digester +} + +// NewReader returns a digesting reader for r. +func NewReader(r io.Reader, alg Algorithm) (*Reader, error) { + if r == nil { + return nil, ErrReaderInvalid + } + d, err := alg.New() + if err != nil { + return nil, err + } + return &Reader{r: r, d: d}, nil +} + +// Read reads from the underlying reader and updates the digest. +func (r *Reader) Read(p []byte) (int, error) { + if r == nil || r.r == nil || r.d == nil { + return 0, ErrReaderInvalid + } + n, err := r.r.Read(p) + if n > 0 { + if _, writeErr := r.d.Write(p[:n]); writeErr != nil { + err = errors.Join(err, writeErr) + } + } + return n, err +} + +// Digest returns the digest of bytes read so far. +func (r *Reader) Digest() (Digest, error) { + if r == nil || r.d == nil { + return Digest{}, ErrReaderInvalid + } + return r.d.Digest() +} + +// State returns the current digest state. +func (r *Reader) State() (State, error) { + if r == nil || r.d == nil { + return State{}, ErrReaderInvalid + } + return r.d.State() +} + +// Size returns the number of bytes read and digested. +func (r *Reader) Size() int64 { + if r == nil || r.d == nil { + return 0 + } + return r.d.Size() +} diff --git a/ocidigest/state.go b/ocidigest/state.go new file mode 100644 index 0000000..f204b93 --- /dev/null +++ b/ocidigest/state.go @@ -0,0 +1,82 @@ +package ocidigest + +import ( + "encoding" + "fmt" + "hash" +) + +const stateEncodingBinaryMarshaler = "binary-marshaler-v1" + +// State is a serialized digest state for short-term resume. +type State struct { + Algorithm string `json:"algorithm"` + Encoding string `json:"encoding"` + Offset int64 `json:"offset"` + Payload []byte `json:"payload"` +} + +// NewFromState restores a digester from st. +func NewFromState(st State) (Digester, error) { + info, err := lookupAlgorithm(st.Algorithm) + if err != nil { + return nil, err + } + if st.Offset < 0 { + return nil, fmt.Errorf("%w: negative offset %d", ErrStateInvalid, st.Offset) + } + if info.newState != nil { + h, err := info.newState(st) + if err != nil { + return nil, err + } + if err := validateRestoredHash(info, h); err != nil { + return nil, err + } + return &digester{info: info, hash: h, size: st.Offset}, nil + } + if st.Encoding != stateEncodingBinaryMarshaler { + return nil, fmt.Errorf("%w: %q", ErrStateUnsupported, st.Encoding) + } + h := info.newHash() + u, ok := h.(encoding.BinaryUnmarshaler) + if !ok { + return nil, fmt.Errorf("%w: %q", ErrStateUnsupported, info.name) + } + if err := u.UnmarshalBinary(st.Payload); err != nil { + return nil, fmt.Errorf("%w: %v", ErrStateInvalid, err) + } + return &digester{info: info, hash: h, size: st.Offset}, nil +} + +func marshalHashState(info algorithmInfo, h hash.Hash) (string, []byte, error) { + if info.marshalState != nil { + encoding, payload, err := info.marshalState(h) + if err != nil { + return "", nil, err + } + if encoding == "" { + return "", nil, fmt.Errorf("%w: empty state encoding", ErrStateInvalid) + } + return encoding, payload, nil + } + m, ok := h.(encoding.BinaryMarshaler) + if !ok { + return "", nil, ErrStateUnsupported + } + payload, err := m.MarshalBinary() + if err != nil { + return "", nil, fmt.Errorf("%w: %v", ErrStateInvalid, err) + } + return stateEncodingBinaryMarshaler, payload, nil +} + +func validateRestoredHash(info algorithmInfo, h hash.Hash) error { + if h == nil { + return ErrHashInvalid + } + if h.Size() != info.size { + return fmt.Errorf("%w: %q has hash size %d, want %d", ErrHashInvalid, info.name, h.Size(), info.size) + } + return nil +} diff --git a/ocidigest/verifier.go b/ocidigest/verifier.go new file mode 100644 index 0000000..1b00737 --- /dev/null +++ b/ocidigest/verifier.go @@ -0,0 +1,140 @@ +package ocidigest + +import ( + "fmt" + "io" +) + +// Verifier verifies bytes against an expected digest. +type Verifier interface { + io.Writer + Digest() Digest + Calculated() (Digest, error) + Verified() bool +} + +type verifier struct { + expected Digest + d Digester +} + +// NewVerifier returns a verifier for expected. +func NewVerifier(expected Digest) (Verifier, error) { + if err := expected.Validate(); err != nil { + return nil, err + } + d, err := expected.Algorithm().New() + if err != nil { + return nil, err + } + return &verifier{expected: expected, d: d}, nil +} + +func (v *verifier) Write(p []byte) (int, error) { + if v == nil || v.d == nil { + return 0, ErrHashInvalid + } + return v.d.Write(p) +} + +func (v *verifier) Digest() Digest { + if v == nil { + return Digest{} + } + return v.expected +} + +func (v *verifier) Calculated() (Digest, error) { + if v == nil || v.d == nil { + return Digest{}, ErrHashInvalid + } + return v.d.Digest() +} + +func (v *verifier) Verified() bool { + got, err := v.Calculated() + return err == nil && !v.expected.IsZero() && got == v.expected +} + +// VerifierReader wraps an io.Reader and verifies the stream at EOF. +type VerifierReader struct { + r io.Reader + v Verifier + final bool + verified bool +} + +// NewVerifierReader returns a reader that verifies bytes read from r. +func NewVerifierReader(r io.Reader, expected Digest) (*VerifierReader, error) { + if r == nil { + return nil, ErrReaderInvalid + } + v, err := NewVerifier(expected) + if err != nil { + return nil, err + } + return &VerifierReader{r: r, v: v}, nil +} + +// Read reads from the underlying reader and updates the verifier. +func (r *VerifierReader) Read(p []byte) (int, error) { + if r == nil || r.r == nil || r.v == nil { + return 0, ErrReaderInvalid + } + if r.final { + return 0, r.finalErr() + } + n, err := r.r.Read(p) + if n > 0 { + if _, writeErr := r.v.Write(p[:n]); writeErr != nil { + return n, writeErr + } + if err == io.EOF { + r.final = true + return n, nil + } + return n, err + } + if err == io.EOF { + r.final = true + return 0, r.finalErr() + } + return n, err +} + +// Digest returns the expected digest. +func (r *VerifierReader) Digest() Digest { + if r == nil || r.v == nil { + return Digest{} + } + return r.v.Digest() +} + +// Calculated returns the digest calculated from bytes read so far. +func (r *VerifierReader) Calculated() (Digest, error) { + if r == nil || r.v == nil { + return Digest{}, ErrReaderInvalid + } + return r.v.Calculated() +} + +// Verified reports whether EOF has been reached and the digest matched. +func (r *VerifierReader) Verified() bool { + if r == nil { + return false + } + return r.verified +} + +func (r *VerifierReader) finalErr() error { + if r.v.Verified() { + r.verified = true + return io.EOF + } + r.verified = false + got, err := r.v.Calculated() + if err != nil { + return err + } + return fmt.Errorf("%w: expected %s, got %s", ErrDigestMismatch, r.v.Digest(), got) +} diff --git a/ocidigest/writer.go b/ocidigest/writer.go new file mode 100644 index 0000000..925c918 --- /dev/null +++ b/ocidigest/writer.go @@ -0,0 +1,63 @@ +package ocidigest + +import ( + "errors" + "io" +) + +// Writer wraps an io.Writer and records the digest of bytes written through it. +type Writer struct { + w io.Writer + d Digester +} + +// NewWriter returns a digesting writer. If w is nil, writes are only digested. +func NewWriter(w io.Writer, alg Algorithm) (*Writer, error) { + d, err := alg.New() + if err != nil { + return nil, err + } + return &Writer{w: w, d: d}, nil +} + +// Write writes p to the underlying writer, if any, and updates the digest. +func (w *Writer) Write(p []byte) (int, error) { + if w == nil || w.d == nil { + return 0, ErrWriterInvalid + } + n := len(p) + var err error + if w.w != nil { + n, err = w.w.Write(p) + } + if n > 0 { + if _, writeErr := w.d.Write(p[:n]); writeErr != nil { + err = errors.Join(err, writeErr) + } + } + return n, err +} + +// Digest returns the digest of bytes written so far. +func (w *Writer) Digest() (Digest, error) { + if w == nil || w.d == nil { + return Digest{}, ErrWriterInvalid + } + return w.d.Digest() +} + +// State returns the current digest state. +func (w *Writer) State() (State, error) { + if w == nil || w.d == nil { + return State{}, ErrWriterInvalid + } + return w.d.State() +} + +// Size returns the number of bytes written and digested. +func (w *Writer) Size() int64 { + if w == nil || w.d == nil { + return 0 + } + return w.d.Size() +} diff --git a/ocifilter/immutable.go b/ocifilter/immutable.go index 0cc5a47..ee47125 100644 --- a/ocifilter/immutable.go +++ b/ocifilter/immutable.go @@ -21,7 +21,7 @@ import ( "fmt" "github.com/docker/oci" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" ) // Immutable returns a registry wrap r but only allows content to be @@ -44,10 +44,10 @@ func (r immutable) PushManifest(ctx context.Context, repo string, contents []byt return r.Interface.PushManifest(ctx, repo, contents, mediaType, params) } var dig oci.Digest - if params != nil && params.Digest != "" { + if params != nil && !params.Digest.IsZero() { dig = params.Digest } else { - dig = digest.FromBytes(contents) + dig = ocidigest.FromBytes(contents) } for _, tag := range tags { diff --git a/ocifilter/select_test.go b/ocifilter/select_test.go index 68e3ca8..172884a 100644 --- a/ocifilter/select_test.go +++ b/ocifilter/select_test.go @@ -21,13 +21,15 @@ import ( "testing" "github.com/docker/oci" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/docker/oci/ocimem" ) +var testDigest = ocidigest.FromBytes([]byte("test")) + func TestAccessCheckerErrorReturn(t *testing.T) { ctx := context.Background() testErr := errors.New("some error") @@ -59,13 +61,13 @@ func TestAccessCheckerAccessRequest(t *testing.T) { assertAccess([]accessCheck{ {"foo/read", AccessRead}, }, func(ctx context.Context, r oci.Interface) error { - _, err := r.GetBlob(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + _, err := r.GetBlob(ctx, "foo/read", testDigest) return err }) assertAccess([]accessCheck{ {"foo/read", AccessRead}, }, func(ctx context.Context, r oci.Interface) error { - rd, err := r.GetBlobRange(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 100, 200) + rd, err := r.GetBlobRange(ctx, "foo/read", testDigest, 100, 200) if rd != nil { rd.Close() } @@ -75,7 +77,7 @@ func TestAccessCheckerAccessRequest(t *testing.T) { assertAccess([]accessCheck{ {"foo/read", AccessRead}, }, func(ctx context.Context, r oci.Interface) error { - rd, err := r.GetManifest(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + rd, err := r.GetManifest(ctx, "foo/read", testDigest) if rd != nil { rd.Close() } @@ -95,14 +97,14 @@ func TestAccessCheckerAccessRequest(t *testing.T) { assertAccess([]accessCheck{ {"foo/read", AccessRead}, }, func(ctx context.Context, r oci.Interface) error { - _, err := r.ResolveBlob(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + _, err := r.ResolveBlob(ctx, "foo/read", testDigest) return err }) assertAccess([]accessCheck{ {"foo/read", AccessRead}, }, func(ctx context.Context, r oci.Interface) error { - _, err := r.ResolveManifest(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + _, err := r.ResolveManifest(ctx, "foo/read", testDigest) return err }) @@ -118,7 +120,7 @@ func TestAccessCheckerAccessRequest(t *testing.T) { }, func(ctx context.Context, r oci.Interface) error { _, err := r.PushBlob(ctx, "foo/write", oci.Descriptor{ MediaType: "application/json", - Digest: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: testDigest, Size: 3, }, strings.NewReader("foo")) return err @@ -146,7 +148,7 @@ func TestAccessCheckerAccessRequest(t *testing.T) { if _, err := w.Write(data); err != nil { return err } - _, err = w.Commit(digest.FromBytes(data)) + _, err = w.Commit(ocidigest.FromBytes(data)) return err }) @@ -154,7 +156,7 @@ func TestAccessCheckerAccessRequest(t *testing.T) { {"foo/read", AccessRead}, {"foo/write", AccessWrite}, }, func(ctx context.Context, r oci.Interface) error { - _, err := r.MountBlob(ctx, "foo/read", "foo/write", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + _, err := r.MountBlob(ctx, "foo/read", "foo/write", testDigest) return err }) @@ -170,13 +172,13 @@ func TestAccessCheckerAccessRequest(t *testing.T) { assertAccess([]accessCheck{ {"foo/write", AccessDelete}, }, func(ctx context.Context, r oci.Interface) error { - return r.DeleteBlob(ctx, "foo/write", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + return r.DeleteBlob(ctx, "foo/write", testDigest) }) assertAccess([]accessCheck{ {"foo/write", AccessDelete}, }, func(ctx context.Context, r oci.Interface) error { - return r.DeleteManifest(ctx, "foo/write", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + return r.DeleteManifest(ctx, "foo/write", testDigest) }) assertAccess([]accessCheck{ @@ -202,7 +204,7 @@ func TestAccessCheckerAccessRequest(t *testing.T) { assertAccess([]accessCheck{ {"foo/read", AccessList}, }, func(ctx context.Context, r oci.Interface) error { - _, err := oci.All(r.Referrers(ctx, "foo/read", "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", nil)) + _, err := oci.All(r.Referrers(ctx, "foo/read", testDigest, nil)) return err }) } diff --git a/ocifilter/sub_test.go b/ocifilter/sub_test.go index d2f317f..673d5c6 100644 --- a/ocifilter/sub_test.go +++ b/ocifilter/sub_test.go @@ -26,8 +26,6 @@ import ( "github.com/docker/oci/ociauth" "github.com/docker/oci/ocimem" "github.com/docker/oci/ocitest" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" ) @@ -40,14 +38,14 @@ func TestSub(t *testing.T) { "b1": "hello", "scratch": "{}", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m1": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "scratch", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("scratch"), + }), Layers: []oci.Descriptor{{ - Digest: "b1", + Digest: ocitest.DigestRef("b1"), }}, }, }, @@ -60,12 +58,12 @@ func TestSub(t *testing.T) { Blobs: map[string]string{ "scratch": "{}", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m1": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "scratch", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("scratch"), + }), }, }, Tags: map[string]string{ @@ -76,12 +74,12 @@ func TestSub(t *testing.T) { Blobs: map[string]string{ "scratch": "{}", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m1": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "scratch", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("scratch"), + }), }, }, Tags: map[string]string{ @@ -117,7 +115,7 @@ func TestSubMaintainsAuthScope(t *testing.T) { // we use the GetBlob entry point as a proxy for testing all the entry points. // TODO it would be nice to have a reusable way (in ocitest, probably) of testing general properties // across all oci.Interface methods. - _, _ = r.GetBlob(ctx, "some/repo", "sha256:fffff") + _, _ = r.GetBlob(ctx, "some/repo", ocitest.DigestRef("scope")) wantScope := ociauth.ParseScope( "other registry:catalog:* repository:foo/bar/a/b:pull,push repository:foo/bar/foo:delete,push", ) @@ -134,11 +132,11 @@ func (r contextChecker) GetBlob(ctx context.Context, repo string, digest oci.Dig return nil, fmt.Errorf("nope") } -func getManifest(t *testing.T, r oci.Interface, repo string, dg digest.Digest) oci.Manifest { +func getManifest(t *testing.T, r oci.Interface, repo string, dg oci.Digest) oci.IndexOrManifest { rd, err := r.GetManifest(context.Background(), repo, dg) require.NoError(t, err) defer rd.Close() - var m oci.Manifest + var m oci.IndexOrManifest data, err := io.ReadAll(rd) require.NoError(t, err) err = json.Unmarshal(data, &m) @@ -146,7 +144,7 @@ func getManifest(t *testing.T, r oci.Interface, repo string, dg digest.Digest) o return m } -func getBlob(t *testing.T, r oci.Interface, repo string, dg digest.Digest) []byte { +func getBlob(t *testing.T, r oci.Interface, repo string, dg oci.Digest) []byte { rd, err := r.GetBlob(context.Background(), repo, dg) require.NoError(t, err) defer rd.Close() @@ -154,3 +152,7 @@ func getBlob(t *testing.T, r oci.Interface, repo string, dg digest.Digest) []byt require.NoError(t, err) return data } + +func ref[T any](x T) *T { + return &x +} diff --git a/ocilarge/download.go b/ocilarge/download.go index 215b4c7..126b9f8 100644 --- a/ocilarge/download.go +++ b/ocilarge/download.go @@ -3,13 +3,12 @@ package ocilarge import ( "context" "fmt" - "hash" "io" "sync" "time" "github.com/docker/oci" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" ) // Tuning constants for the adaptive parallel download pipeline. @@ -88,9 +87,10 @@ func DownloadLargeBlob(ctx context.Context, reg oci.Interface, repo string, dgst go runPipeline(ctx, reg, repo, dgst, desc.Size, chunkSize, probeData, pw) + digester, _ := desc.Digest.Algorithm().New() return &blobReader{ r: pr, - digester: desc.Digest.Algorithm().Hash(), + digester: digester, desc: desc, verify: true, }, nil @@ -310,9 +310,10 @@ func downloadSingle(ctx context.Context, reg oci.Interface, repo string, dgst oc } pw.Close() }() + digester, _ := desc.Digest.Algorithm().New() return &blobReader{ r: pr, - digester: desc.Digest.Algorithm().Hash(), + digester: digester, desc: desc, verify: true, }, nil @@ -321,7 +322,7 @@ func downloadSingle(ctx context.Context, reg oci.Interface, repo string, dgst oc type blobReader struct { r io.ReadCloser n int64 - digester hash.Hash + digester ocidigest.Digester desc oci.Descriptor verify bool } @@ -333,7 +334,9 @@ func (r *blobReader) Descriptor() oci.Descriptor { func (r *blobReader) Read(buf []byte) (int, error) { n, err := r.r.Read(buf) r.n += int64(n) - r.digester.Write(buf[:n]) + if _, writeErr := r.digester.Write(buf[:n]); writeErr != nil { + return n, writeErr + } if err == nil { if r.n > r.desc.Size { // Fail early when the blob is too big; we can do that even @@ -351,7 +354,10 @@ func (r *blobReader) Read(buf []byte) (int, error) { if r.n != r.desc.Size { return n, fmt.Errorf("blob size mismatch (%d/%d): %w", r.n, r.desc.Size, oci.ErrSizeInvalid) } - gotDigest := digest.NewDigest(r.desc.Digest.Algorithm(), r.digester) + gotDigest, err := r.digester.Digest() + if err != nil { + return n, err + } if gotDigest != r.desc.Digest { return n, fmt.Errorf("digest mismatch when reading blob") } diff --git a/ocilarge/upload.go b/ocilarge/upload.go index a10cc46..9be71db 100644 --- a/ocilarge/upload.go +++ b/ocilarge/upload.go @@ -6,15 +6,36 @@ import ( "io" "github.com/docker/oci" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" ) +const defaultUploadChunkSize = 100 * 1024 * 1024 // 100 MB + +// UploadLargeBlobParameters holds optional parameters for UploadLargeBlob. +type UploadLargeBlobParameters struct { + // ChunkSize is the maximum number of bytes to upload at a time. + // If zero or negative, a default of 100 MB is used. + ChunkSize int + + // Algorithm is the digest algorithm used to commit the blob. + // If zero, ocidigest.Canonical is used. + Algorithm ocidigest.Algorithm +} + // UploadLargeBlob uploads a large blob in chunks with retries so that uploads can be resumed in case of network error -func UploadLargeBlob(ctx context.Context, reg oci.Interface, repo string, f io.ReadCloser, chunkSize int) (oci.Descriptor, error) { +func UploadLargeBlob(ctx context.Context, reg oci.Interface, repo string, f io.ReadCloser, params *UploadLargeBlobParameters) (oci.Descriptor, error) { defer f.Close() - if chunkSize <= 0 { - chunkSize = 100 * 1024 * 1024 // 100 MB + chunkSize := defaultUploadChunkSize + algorithm := ocidigest.Canonical + if params != nil { + if params.ChunkSize > 0 { + chunkSize = params.ChunkSize + } + if params.Algorithm.String() != "" { + algorithm = params.Algorithm + } } + bw, err := reg.PushBlobChunked(ctx, repo, chunkSize) if err != nil { return oci.Descriptor{}, fmt.Errorf("starting chunked upload: %w", err) @@ -22,11 +43,16 @@ func UploadLargeBlob(ctx context.Context, reg oci.Interface, repo string, f io.R defer func() { _ = bw.Cancel() }() // no-op after a successful Commit buf := make([]byte, chunkSize) - dgstr := digest.Canonical.Digester() + dgstr, err := algorithm.New() + if err != nil { + return oci.Descriptor{}, err + } for { n, readErr := io.ReadFull(f, buf) if n > 0 { - dgstr.Hash().Write(buf[:n]) + if _, err := dgstr.Write(buf[:n]); err != nil { + return oci.Descriptor{}, err + } var writeErr error for range 3 { // try writing each chunk three times @@ -50,5 +76,9 @@ func UploadLargeBlob(ctx context.Context, reg oci.Interface, repo string, f io.R return oci.Descriptor{}, fmt.Errorf("reading source: %w", readErr) } } - return bw.Commit(dgstr.Digest()) + dgst, err := dgstr.Digest() + if err != nil { + return oci.Descriptor{}, err + } + return bw.Commit(dgst) } diff --git a/ocimem/blob.go b/ocimem/blob.go index 5607409..25cf09e 100644 --- a/ocimem/blob.go +++ b/ocimem/blob.go @@ -21,7 +21,7 @@ import ( "sync" "github.com/docker/oci" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" ) // NewBytesReader returns an implementation of oci.BlobReader @@ -181,7 +181,7 @@ func (b *Buffer) checkCommit(dig oci.Digest) (err error) { b.commitErr = err } }() - if digest.FromBytes(b.buf) != dig { + if ocidigest.FromBytes(b.buf) != dig { return fmt.Errorf("digest mismatch (sha256(%q) != %s): %w", b.buf, dig, oci.ErrDigestInvalid) } b.desc = oci.Descriptor{ diff --git a/ocimem/check_test.go b/ocimem/check_test.go index fda3215..1fe070d 100644 --- a/ocimem/check_test.go +++ b/ocimem/check_test.go @@ -8,8 +8,6 @@ import ( "github.com/docker/oci" "github.com/docker/oci/ocitest" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" ) @@ -23,15 +21,15 @@ var pushManifestTests = []struct { wantError string }{{ testName: "NonExistentConfigReference", - mediaType: ocispec.MediaTypeImageManifest, + mediaType: oci.MediaTypeImageManifest, manifestData: func(ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ MediaType: "application/something", Size: 1, - Digest: digest.FromString("a"), - }, + Digest: ocitest.DigestRef("a"), + }), }) }, wantError: `invalid manifest: blob for config not found`, @@ -42,15 +40,15 @@ var pushManifestTests = []struct { "a": "{}", }, }, - mediaType: ocispec.MediaTypeImageManifest, + mediaType: oci.MediaTypeImageManifest, manifestData: func(content ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, - Config: content.Blobs["a"], + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, + Config: ref(content.Blobs["a"]), Layers: []oci.Descriptor{{ MediaType: "application/something", Size: 1, - Digest: digest.FromString("b"), + Digest: ocitest.DigestRef("b"), }}, }) }, @@ -62,29 +60,29 @@ var pushManifestTests = []struct { "a": "{}", }, }, - mediaType: ocispec.MediaTypeImageManifest, + mediaType: oci.MediaTypeImageManifest, manifestData: func(content ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, - Config: content.Blobs["a"], + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, + Config: ref(content.Blobs["a"]), Subject: &oci.Descriptor{ MediaType: "application/something", Size: 1, - Digest: digest.FromString("b"), + Digest: ocitest.DigestRef("b"), }, }) }, // Non-existent subject references are explicitly allowed. }, { testName: "NonExistentImageIndexManifestReference", - mediaType: ocispec.MediaTypeImageIndex, + mediaType: oci.MediaTypeImageIndex, manifestData: func(content ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(ocispec.Index{ - MediaType: ocispec.MediaTypeImageIndex, + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageIndex, Manifests: []oci.Descriptor{{ - MediaType: ocispec.MediaTypeImageManifest, + MediaType: oci.MediaTypeImageManifest, Size: 1, - Digest: digest.FromString("a"), + Digest: ocitest.DigestRef("a"), }}, }) }, @@ -99,28 +97,28 @@ var pushManifestTests = []struct { config: Config{ LaxChildReferences: true, }, - mediaType: ocispec.MediaTypeImageManifest, + mediaType: oci.MediaTypeImageManifest, manifestData: func(content ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, - Config: content.Blobs["a"], + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, + Config: ref(content.Blobs["a"]), Layers: []oci.Descriptor{{ MediaType: "application/something", Size: 1, - Digest: digest.FromString("b"), + Digest: ocitest.DigestRef("b"), }}, }) }, }, { testName: "NonExistentImageIndexSubjectReference", - mediaType: ocispec.MediaTypeImageIndex, + mediaType: oci.MediaTypeImageIndex, manifestData: func(content ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(ocispec.Index{ - MediaType: ocispec.MediaTypeImageIndex, + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageIndex, Subject: &oci.Descriptor{ MediaType: "application/something", Size: 1, - Digest: digest.FromString("b"), + Digest: ocitest.DigestRef("b"), }, }) }, @@ -132,14 +130,14 @@ var pushManifestTests = []struct { "a": "{}", "b": "other", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), Layers: []oci.Descriptor{{ - Digest: "a", + Digest: ocitest.DigestRef("a"), }}, }, }, @@ -150,12 +148,12 @@ var pushManifestTests = []struct { config: Config{ ImmutableTags: true, }, - mediaType: ocispec.MediaTypeImageManifest, + mediaType: oci.MediaTypeImageManifest, tag: "sometag", manifestData: func(content ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, - Config: content.Blobs["a"], + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, + Config: ref(content.Blobs["a"]), Layers: []oci.Descriptor{content.Blobs["a"]}, Annotations: map[string]string{ "different": "thing", @@ -170,14 +168,14 @@ var pushManifestTests = []struct { "a": "{}", "b": "other", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), Layers: []oci.Descriptor{{ - Digest: "a", + Digest: ocitest.DigestRef("a"), }}, }, }, @@ -188,7 +186,7 @@ var pushManifestTests = []struct { config: Config{ ImmutableTags: true, }, - mediaType: ocispec.MediaTypeImageManifest, + mediaType: oci.MediaTypeImageManifest, tag: "sometag", manifestData: func(content ocitest.PushedRepoContent) []byte { return content.ManifestData["m"] @@ -200,14 +198,14 @@ var pushManifestTests = []struct { "a": "{}", "b": "other", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), Layers: []oci.Descriptor{{ - Digest: "a", + Digest: ocitest.DigestRef("a"), }}, }, }, @@ -231,14 +229,14 @@ var pushManifestTests = []struct { "a": "{}", "b": "other", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), Layers: []oci.Descriptor{{ - Digest: "a", + Digest: ocitest.DigestRef("a"), }}, }, }, @@ -246,12 +244,12 @@ var pushManifestTests = []struct { "sometag": "m", }, }, - mediaType: ocispec.MediaTypeImageManifest, + mediaType: oci.MediaTypeImageManifest, tag: "sometag", manifestData: func(content ocitest.PushedRepoContent) []byte { - return mustJSONMarshal(oci.Manifest{ - MediaType: ocispec.MediaTypeImageManifest, - Config: content.Blobs["a"], + return mustJSONMarshal(oci.IndexOrManifest{ + MediaType: oci.MediaTypeImageManifest, + Config: ref(content.Blobs["a"]), Layers: []oci.Descriptor{content.Blobs["a"]}, Annotations: map[string]string{ "different": "thing", @@ -295,7 +293,7 @@ var deleteBlobTests = []struct { }{{ testName: "NonExistentRepo", getDigest: func(content ocitest.PushedRepoContent) oci.Digest { - return digest.FromString("blshdfsvg") + return ocitest.DigestRef("blshdfsvg") }, wantError: "name unknown: repository name not known to registry", }, { @@ -306,7 +304,7 @@ var deleteBlobTests = []struct { }, }, getDigest: func(content ocitest.PushedRepoContent) oci.Digest { - return digest.FromString("blshdfsvg") + return ocitest.DigestRef("blshdfsvg") }, wantError: "blob unknown: blob unknown to registry", }, { @@ -318,14 +316,14 @@ var deleteBlobTests = []struct { Blobs: map[string]string{ "a": "{}", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), Layers: []oci.Descriptor{{ - Digest: "a", + Digest: ocitest.DigestRef("a"), }}, }, }, @@ -347,20 +345,20 @@ var deleteBlobTests = []struct { "a": "{}", "b": "other", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m0": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), }, "m1": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "b", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("b"), + }), Subject: &oci.Descriptor{ - Digest: "m0", + Digest: ocitest.DigestRef("m0"), }, }, }, @@ -409,7 +407,7 @@ var deleteManifestTests = []struct { }{{ testName: "NonExistentRepo", getDigest: func(content ocitest.PushedRepoContent) oci.Digest { - return digest.FromString("blshdfsvg") + return ocitest.DigestRef("blshdfsvg") }, wantError: "name unknown: repository name not known to registry", }, { @@ -420,7 +418,7 @@ var deleteManifestTests = []struct { }, }, getDigest: func(content ocitest.PushedRepoContent) oci.Digest { - return digest.FromString("blshdfsvg") + return ocitest.DigestRef("blshdfsvg") }, wantError: "manifest unknown: manifest unknown to registry", }, { @@ -432,12 +430,12 @@ var deleteManifestTests = []struct { Blobs: map[string]string{ "a": "{}", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), }, }, Tags: map[string]string{ @@ -458,20 +456,20 @@ var deleteManifestTests = []struct { "a": "{}", "b": "other", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m0": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), }, "m1": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "b", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("b"), + }), Subject: &oci.Descriptor{ - Digest: "m0", + Digest: ocitest.DigestRef("m0"), }, }, }, @@ -539,12 +537,12 @@ var deleteTagTests = []struct { Blobs: map[string]string{ "a": "{}", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), }, }, Tags: map[string]string{ @@ -560,20 +558,20 @@ var deleteTagTests = []struct { "a": "{}", "b": "other", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m0": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), }, "m1": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "b", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("b"), + }), Subject: &oci.Descriptor{ - Digest: "m0", + Digest: ocitest.DigestRef("m0"), }, }, }, @@ -623,6 +621,10 @@ func mustJSONMarshal(x any) []byte { return data } +func ref[T any](x T) *T { + return &x +} + func TestTagsLimit(t *testing.T) { ctx := context.Background() r := ocitest.NewRegistry(t, New()) @@ -631,12 +633,12 @@ func TestTagsLimit(t *testing.T) { Blobs: map[string]string{ "a": "{}", }, - Manifests: map[string]oci.Manifest{ + Manifests: map[string]oci.IndexOrManifest{ "m": { - MediaType: ocispec.MediaTypeImageManifest, - Config: oci.Descriptor{ - Digest: "a", - }, + MediaType: oci.MediaTypeImageManifest, + Config: ref(oci.Descriptor{ + Digest: ocitest.DigestRef("a"), + }), }, }, Tags: map[string]string{ diff --git a/ocimem/desciter.go b/ocimem/desciter.go index 8b2a100..e449fa0 100644 --- a/ocimem/desciter.go +++ b/ocimem/desciter.go @@ -7,7 +7,6 @@ import ( "iter" "github.com/docker/oci" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) type refKind int @@ -37,24 +36,30 @@ type manifestInfo struct { type descIter = iter.Seq[descInfo] -// TODO support other manifest types. -var manifestInfoByMediaType = map[string]func(data []byte) (manifestInfo, error){ - ocispec.MediaTypeImageManifest: manifestInfoForType(imageInfo), - ocispec.MediaTypeImageIndex: manifestInfoForType(indexInfo), -} - // getManifestInfo returns information on the manifest // described by the given media type and data. func getManifestInfo(mediaType string, data []byte) (manifestInfo, error) { - getInfo := manifestInfoByMediaType[mediaType] - if getInfo == nil { + switch mediaType { + case oci.MediaTypeImageManifest, oci.MediaTypeDockerManifest, + oci.MediaTypeImageIndex, oci.MediaTypeDockerManifestList: + default: // TODO provide a configuration option to disallow unknown manifest types. //return nil, fmt.Errorf("media type %q: %w", mediaType, errUnknownManifestMediaTypeForIteration) return manifestInfo{ descriptors: func(func(descInfo) bool) {}, }, nil } - return getInfo(data) + var m oci.IndexOrManifest + if err := json.Unmarshal(data, &m); err != nil { + return manifestInfo{}, fmt.Errorf("cannot unmarshal into %T: %v", &m, err) + } + if m.MediaType == "" { + m.MediaType = mediaType + } + if err := m.Validate(); err != nil { + return manifestInfo{}, err + } + return indexOrManifestInfo(m), nil } // repoTagIter returns an iterator that iterates through @@ -73,19 +78,18 @@ func repoTagIter(r *repository) descIter { } } -func manifestInfoForType[T any](getInfo func(T) manifestInfo) func(data []byte) (manifestInfo, error) { - return func(data []byte) (manifestInfo, error) { - var x T - if err := json.Unmarshal(data, &x); err != nil { - return manifestInfo{}, fmt.Errorf("cannot unmarshal into %T: %v", &x, err) - } - return getInfo(x), nil - } -} - -func imageInfo(m oci.Manifest) manifestInfo { +func indexOrManifestInfo(m oci.IndexOrManifest) manifestInfo { var info manifestInfo info.descriptors = func(yield func(descInfo) bool) { + for i, manifest := range m.Manifests { + if !yield(descInfo{ + name: fmt.Sprintf("manifests[%d]", i), + kind: kindManifest, + desc: manifest, + }) { + return + } + } for i, layer := range m.Layers { if !yield(descInfo{ name: fmt.Sprintf("layers[%d]", i), @@ -95,12 +99,14 @@ func imageInfo(m oci.Manifest) manifestInfo { return } } - if !yield(descInfo{ - name: "config", - desc: m.Config, - kind: kindBlob, - }) { - return + if m.Config != nil { + if !yield(descInfo{ + name: "config", + desc: *m.Config, + kind: kindBlob, + }) { + return + } } if m.Subject != nil { if !yield(descInfo{ @@ -122,37 +128,11 @@ func imageInfo(m oci.Manifest) manifestInfo { // missing in an index, the artifactType MUST be omitted. The // descriptors MUST include annotations from the image manifest or // index. - info.artifactType = cmp.Or(m.ArtifactType, m.Config.MediaType) - info.annotations = m.Annotations - if m.Subject != nil { - info.subject = m.Subject.Digest - } - return info -} - -func indexInfo(m ocispec.Index) manifestInfo { - var info manifestInfo - info.descriptors = func(yield func(descInfo) bool) { - for i, manifest := range m.Manifests { - if !yield(descInfo{ - name: fmt.Sprintf("manifests[%d]", i), - kind: kindManifest, - desc: manifest, - }) { - return - } - } - if m.Subject != nil { - if !yield(descInfo{ - name: "subject", - kind: kindSubjectManifest, - desc: *m.Subject, - }) { - return - } - } + if m.Config != nil { + info.artifactType = cmp.Or(m.ArtifactType, m.Config.MediaType) + } else { + info.artifactType = m.ArtifactType } - info.artifactType = m.ArtifactType // Note: no config descriptor to fall back to. info.annotations = m.Annotations if m.Subject != nil { info.subject = m.Subject.Digest diff --git a/ocimem/lister.go b/ocimem/lister.go index 1de86f3..6d004c7 100644 --- a/ocimem/lister.go +++ b/ocimem/lister.go @@ -86,5 +86,5 @@ func mapKeysIter[K comparable, V any](m map[K]V, cmp func(K, K) int, startAfter } func compareDescriptor(d0, d1 oci.Descriptor) int { - return strings.Compare(string(d0.Digest), string(d1.Digest)) + return strings.Compare(d0.Digest.String(), d1.Digest.String()) } diff --git a/ocimem/registry.go b/ocimem/registry.go index 2240b06..1af2fab 100644 --- a/ocimem/registry.go +++ b/ocimem/registry.go @@ -21,8 +21,8 @@ import ( "sync" "github.com/docker/oci" + "github.com/docker/oci/ocidigest" "github.com/docker/oci/ociref" - "github.com/opencontainers/go-digest" ) var _ oci.Interface = (*Registry)(nil) @@ -52,7 +52,7 @@ func (b *blob) descriptor() oci.Descriptor { return oci.Descriptor{ MediaType: b.mediaType, Size: int64(len(b.data)), - Digest: digest.FromBytes(b.data), + Digest: ocidigest.FromBytes(b.data), ArtifactType: b.info.artifactType, Annotations: b.info.annotations, } @@ -142,8 +142,8 @@ func (r *Registry) makeRepo(repoName string) (*repository, error) { } repo := &repository{ tags: make(map[string]oci.Descriptor), - manifests: make(map[digest.Digest]*blob), - blobs: make(map[digest.Digest]*blob), + manifests: make(map[oci.Digest]*blob), + blobs: make(map[oci.Digest]*blob), uploads: make(map[string]*Buffer), } r.repos[repoName] = repo @@ -151,7 +151,7 @@ func (r *Registry) makeRepo(repoName string) (*repository, error) { } // SHA256("") -const emptyHash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +var emptyHash = ocidigest.FromBytes(nil) // CheckDescriptor checks that the given descriptor matches the given data or, // if data is nil, that the descriptor looks sane. @@ -160,7 +160,7 @@ func CheckDescriptor(desc oci.Descriptor, data []byte) error { return fmt.Errorf("invalid digest: %v", err) } if data != nil { - if digest.FromBytes(data) != desc.Digest { + if ocidigest.FromBytes(data) != desc.Digest { return fmt.Errorf("digest mismatch") } if desc.Size != int64(len(data)) { diff --git a/ocimem/writer.go b/ocimem/writer.go index 3a343ac..546925f 100644 --- a/ocimem/writer.go +++ b/ocimem/writer.go @@ -21,8 +21,8 @@ import ( "slices" "github.com/docker/oci" + "github.com/docker/oci/ocidigest" "github.com/docker/oci/ociref" - "github.com/opencontainers/go-digest" ) // This file implements the oci.Writer methods. @@ -108,12 +108,15 @@ func (r *Registry) PushManifest(ctx context.Context, repoName string, data []byt return oci.Descriptor{}, err } var dig oci.Digest - if params != nil && params.Digest != "" { + if params != nil && !params.Digest.IsZero() { // Validate the provided digest against the contents. if err := params.Digest.Validate(); err != nil { return oci.Descriptor{}, fmt.Errorf("invalid digest: %v: %w", err, oci.ErrDigestInvalid) } - verifier := params.Digest.Verifier() + verifier, err := ocidigest.NewVerifier(params.Digest) + if err != nil { + return oci.Descriptor{}, fmt.Errorf("cannot make digest verifier: %v", err) + } if _, err := verifier.Write(data); err != nil { return oci.Descriptor{}, fmt.Errorf("cannot verify digest: %v", err) } @@ -122,7 +125,7 @@ func (r *Registry) PushManifest(ctx context.Context, repoName string, data []byt } dig = params.Digest } else { - dig = digest.FromBytes(data) + dig = ocidigest.FromBytes(data) } desc := oci.Descriptor{ Digest: dig, @@ -153,7 +156,7 @@ func (r *Registry) PushManifest(ctx context.Context, repoName string, data []byt } // make a copy of the data to avoid potential corruption. data = slices.Clone(data) - if params == nil || params.Digest == "" { + if params == nil || params.Digest.IsZero() { // Only check descriptor with data when using canonical digest, // since CheckDescriptor uses canonical hashing. if err := CheckDescriptor(desc, data); err != nil { diff --git a/ociref/reference.go b/ociref/reference.go index 5c18850..c88b03f 100644 --- a/ociref/reference.go +++ b/ociref/reference.go @@ -22,7 +22,7 @@ import ( "sync" "github.com/docker/oci/internal/ocidocker" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" ) // The following regular expressions derived from code in the @@ -99,7 +99,7 @@ var referencePat = sync.OnceValue(func() *regexp.Regexp { `(?:` + `(` + domainAndPort + `)` + `/` + `)?` + // capture 1: host `(` + repoName + `)` + // capture 2: repository name `(?:` + `:([^@]+))?` + // capture 3: tag; rely on Go logic to test validity. - `(?:` + `@(.+))?` + // capture 4: digest; rely on go-digest to find issues + `(?:` + `@(.+))?` + // capture 4: digest; rely on ocidigest to find issues `)$`, ) }) @@ -131,8 +131,8 @@ type Reference struct { Digest Digest } -// Digest is a content-addressable digest. It is an alias for [digest.Digest]. -type Digest = digest.Digest +// Digest is a content-addressable digest. +type Digest = ocidigest.Digest // IsValidHost reports whether s is a valid host (or host:port) part of a reference string. func IsValidHost(s string) bool { @@ -152,7 +152,7 @@ func IsValidTag(s string) bool { // IsValidDigest reports whether the digest d is well formed. func IsValidDigest(d string) bool { - _, err := digest.Parse(d) + _, err := ocidigest.Parse(d) return err == nil } @@ -187,11 +187,16 @@ func ParseRelative(refStr string) (Reference, error) { return Reference{}, fmt.Errorf("invalid reference syntax (%q)", refStr) } var ref Reference - ref.Host, ref.Repository, ref.Tag, ref.Digest = m[1], m[2], m[3], Digest(m[4]) + ref.Host, ref.Repository, ref.Tag = m[1], m[2], m[3] + var err error + ref.Digest, err = ocidigest.Parse(m[4]) + if err != nil { + return Reference{}, fmt.Errorf("invalid digest %q: %v", m[4], err) + } // Check lengths and digest: we don't check these as part of the regexp // because it's more efficient to do it in Go and we get // nicer error messages as a result. - if len(ref.Digest) > 0 { + if !ref.Digest.IsZero() { if err := ref.Digest.Validate(); err != nil { return Reference{}, fmt.Errorf("invalid digest %q: %v", ref.Digest, err) } @@ -239,7 +244,8 @@ func isWord(c byte) bool { // [HOST/]NAME[:TAG|@DIGEST] func (ref Reference) String() string { var buf strings.Builder - buf.Grow(len(ref.Host) + 1 + len(ref.Repository) + 1 + len(ref.Tag) + 1 + len(ref.Digest)) + digestString := ref.Digest.String() + buf.Grow(len(ref.Host) + 1 + len(ref.Repository) + 1 + len(ref.Tag) + 1 + len(digestString)) if ref.Host != "" { buf.WriteString(ref.Host) buf.WriteByte('/') @@ -249,9 +255,9 @@ func (ref Reference) String() string { buf.WriteByte(':') buf.WriteString(ref.Tag) } - if len(ref.Digest) > 0 { + if digestString != "" { buf.WriteByte('@') - buf.WriteString(string(ref.Digest)) + buf.WriteString(digestString) } return buf.String() } diff --git a/ociref/reference_test.go b/ociref/reference_test.go index cbaf056..2c8ec66 100644 --- a/ociref/reference_test.go +++ b/ociref/reference_test.go @@ -22,10 +22,19 @@ import ( "strings" "testing" + "github.com/docker/oci/ocidigest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func mustParseDigest(s string) Digest { + digest, err := ocidigest.Parse(s) + if err != nil { + panic(err) + } + return digest +} + var parseReferenceTests = []struct { testName string // input is the repository name or name component testcase @@ -92,7 +101,7 @@ var parseReferenceTests = []struct { wantRef: Reference{ Host: "test:5000", Repository: "repo", - Digest: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: mustParseDigest("sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), }, }, { @@ -101,7 +110,7 @@ var parseReferenceTests = []struct { Host: "test:5000", Repository: "repo", Tag: "tag", - Digest: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: mustParseDigest("sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), }, }, { @@ -126,11 +135,11 @@ var parseReferenceTests = []struct { }, { input: "repo@sha256:ffffffffffffffffffffffffffffffffff", - wantErr: `invalid digest "sha256:ffffffffffffffffffffffffffffffffff": invalid checksum digest length`, + wantErr: `invalid digest "sha256:ffffffffffffffffffffffffffffffffff": ocidigest: encoded digest is invalid`, }, { input: "validname@invalidDigest:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - wantErr: `invalid digest "invalidDigest:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff": invalid checksum digest format`, + wantErr: `invalid digest "invalidDigest:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff": ocidigest: invalid algorithm name`, }, { input: "Uppercase:tag", @@ -210,7 +219,7 @@ var parseReferenceTests = []struct { Host: "xn--7o8h.com", Repository: "myimage", Tag: "xn--7o8h.com", - Digest: "sha512:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: mustParseDigest("sha512:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), }, }, { @@ -329,7 +338,7 @@ var parseReferenceTests = []struct { wantRef: Reference{ Host: "[2001:db8::1]:5000", Repository: "repo", - Digest: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: mustParseDigest("sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), }, }, { @@ -338,7 +347,7 @@ var parseReferenceTests = []struct { Host: "[2001:db8::1]:5000", Repository: "repo", Tag: "tag", - Digest: "sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + Digest: mustParseDigest("sha256:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), }, }, { diff --git a/ociserver/deleter.go b/ociserver/deleter.go index 1bc6444..b3ae15f 100644 --- a/ociserver/deleter.go +++ b/ociserver/deleter.go @@ -18,12 +18,16 @@ import ( "context" "net/http" - "github.com/docker/oci" "github.com/docker/oci/internal/ocirequest" + "github.com/docker/oci/ocidigest" ) func (r *registry) handleBlobDelete(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - if err := r.backend.DeleteBlob(ctx, rreq.Repo, oci.Digest(rreq.Digest)); err != nil { + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + if err := r.backend.DeleteBlob(ctx, rreq.Repo, digest); err != nil { return err } resp.WriteHeader(http.StatusAccepted) @@ -35,7 +39,11 @@ func (r *registry) handleManifestDelete(ctx context.Context, resp http.ResponseW if rreq.Tag != "" { err = r.backend.DeleteTag(ctx, rreq.Repo, rreq.Tag) } else { - err = r.backend.DeleteManifest(ctx, rreq.Repo, oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + err = r.backend.DeleteManifest(ctx, rreq.Repo, digest) } if err != nil { return err diff --git a/ociserver/lister.go b/ociserver/lister.go index 7b266e8..cd9818d 100644 --- a/ociserver/lister.go +++ b/ociserver/lister.go @@ -26,9 +26,9 @@ import ( "strconv" "github.com/docker/oci" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/docker/oci/internal/ocirequest" + "github.com/docker/oci/ocidigest" ) const maxPageSize = 10000 @@ -92,9 +92,9 @@ func (r *registry) handleReferrersList(ctx context.Context, resp http.ResponseWr artifactType = "" } - im := &ocispec.Index{ - Versioned: v2, - MediaType: mediaTypeOCIImageIndex, + im := &oci.IndexOrManifest{ + SchemaVersion: 2, + MediaType: mediaTypeOCIImageIndex, } // TODO this could potentially end up with a very large response which we might @@ -102,7 +102,11 @@ func (r *registry) handleReferrersList(ctx context.Context, resp http.ResponseWr // request, linked to the next one with a Link header. However, arranging that is non-trivial // because we'd need a way to return a link value to the client that enables a fresh // call to Referrers to start where the old one left off. For now, we'll punt. - for desc, err := range r.backend.Referrers(ctx, rreq.Repo, oci.Digest(rreq.Digest), &oci.ReferrersParameters{ + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + for desc, err := range r.backend.Referrers(ctx, rreq.Repo, digest, &oci.ReferrersParameters{ ArtifactType: artifactType, }) { if err != nil { diff --git a/ociserver/mediatype.go b/ociserver/mediatype.go index dce8df9..cd5541b 100644 --- a/ociserver/mediatype.go +++ b/ociserver/mediatype.go @@ -14,14 +14,12 @@ package ociserver -import ( - ocispec "github.com/opencontainers/image-spec/specs-go/v1" -) +import "github.com/docker/oci" const ( - mediaTypeOCIImageIndex = ocispec.MediaTypeImageIndex - mediaTypeOCIManifestSchema1 = ocispec.MediaTypeImageManifest - mediaTypeOCIConfigJSON = ocispec.MediaTypeImageConfig + mediaTypeOCIImageIndex = oci.MediaTypeImageIndex + mediaTypeOCIManifestSchema1 = oci.MediaTypeImageManifest + mediaTypeOCIConfigJSON = oci.MediaTypeImageConfig mediaTypeDockerConfigJSON = "application/vnd.docker.container.image.v1+json" mediaTypeOctetStream = "application/octet-stream" ) diff --git a/ociserver/proxy_test.go b/ociserver/proxy_test.go index 1c6a81c..70f8ec8 100644 --- a/ociserver/proxy_test.go +++ b/ociserver/proxy_test.go @@ -23,8 +23,7 @@ import ( "testing" "github.com/docker/oci" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/docker/oci/ocidigest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -64,9 +63,9 @@ var proxyTests = []struct { { name: "PushBlob_small", clientDo: func(ctx context.Context, client oci.Interface) error { - _, err := client.PushBlob(ctx, "foo/bar", ocispec.Descriptor{ + _, err := client.PushBlob(ctx, "foo/bar", oci.Descriptor{ Size: int64(len(smallData)), - Digest: digest.FromBytes(smallData), + Digest: ocidigest.FromBytes(smallData), }, bytes.NewReader(smallData)) return err }, @@ -82,9 +81,9 @@ var proxyTests = []struct { { name: "PushBlob_large", clientDo: func(ctx context.Context, client oci.Interface) error { - _, err := client.PushBlob(ctx, "foo/bar", ocispec.Descriptor{ + _, err := client.PushBlob(ctx, "foo/bar", oci.Descriptor{ Size: int64(len(largeData)), - Digest: digest.FromBytes(largeData), + Digest: ocidigest.FromBytes(largeData), }, bytes.NewReader(largeData)) return err }, @@ -107,7 +106,7 @@ var proxyTests = []struct { if _, err := bw.Write(largeData); err != nil { return err } - if _, err := bw.Commit(digest.FromBytes(largeData)); err != nil { + if _, err := bw.Commit(ocidigest.FromBytes(largeData)); err != nil { return err } return nil diff --git a/ociserver/reader.go b/ociserver/reader.go index 716816e..ba1b2c7 100644 --- a/ociserver/reader.go +++ b/ociserver/reader.go @@ -22,15 +22,20 @@ import ( "github.com/docker/oci" "github.com/docker/oci/internal/ocirequest" + "github.com/docker/oci/ocidigest" ) func (r *registry) handleBlobHead(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - desc, err := r.backend.ResolveBlob(ctx, rreq.Repo, oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + desc, err := r.backend.ResolveBlob(ctx, rreq.Repo, digest) if err != nil { return err } resp.Header().Set("Content-Length", fmt.Sprint(desc.Size)) - resp.Header().Set("Docker-Content-Digest", string(desc.Digest)) + resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) // TODO this is true in theory, but what if the backend doesn't support GetBlobRange ? resp.Header().Set("Accept-Ranges", "bytes") resp.WriteHeader(http.StatusOK) @@ -43,7 +48,11 @@ func (r *registry) handleBlobGet(ctx context.Context, resp http.ResponseWriter, // what to pass back, so resolve the blob first so we don't // stimulate the backend to start sending the whole stream // only to abandon it. - desc, err := r.backend.ResolveBlob(ctx, rreq.Repo, oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + desc, err := r.backend.ResolveBlob(ctx, rreq.Repo, digest) if err != nil { // TODO this might not be the best response because ResolveBlob is // often implemented with a HEAD request that can't return an error @@ -68,7 +77,11 @@ func (r *registry) handleBlobGet(ctx context.Context, resp http.ResponseWriter, } switch len(ranges) { case 0: - blob, err := r.backend.GetBlob(ctx, rreq.Repo, oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + blob, err := r.backend.GetBlob(ctx, rreq.Repo, digest) if err != nil { return err } @@ -83,7 +96,11 @@ func (r *registry) handleBlobGet(ctx context.Context, resp http.ResponseWriter, return nil case 1: rng := ranges[0] - blob, err := r.backend.GetBlobRange(ctx, rreq.Repo, oci.Digest(rreq.Digest), rng.start, rng.end) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + blob, err := r.backend.GetBlobRange(ctx, rreq.Repo, digest, rng.start, rng.end) if err != nil { // TODO fall back to using GetBlob if err is ErrUnsupported? return err @@ -120,14 +137,21 @@ func (r *registry) handleManifestGet(ctx context.Context, resp http.ResponseWrit if rreq.Tag != "" { mr, err = r.backend.GetTag(ctx, rreq.Repo, rreq.Tag) } else { - mr, err = r.backend.GetManifest(ctx, rreq.Repo, oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + mr, err = r.backend.GetManifest(ctx, rreq.Repo, digest) } if err != nil { return err } + if mr == nil { + return fmt.Errorf("backend returned nil manifest reader") + } desc := mr.Descriptor() if !r.opts.OmitDigestFromTagGetResponse { - resp.Header().Set("Docker-Content-Digest", string(desc.Digest)) + resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) } resp.Header().Set("Content-Type", desc.MediaType) resp.Header().Set("Content-Length", fmt.Sprint(desc.Size)) @@ -142,7 +166,11 @@ func (r *registry) handleManifestHead(ctx context.Context, resp http.ResponseWri if rreq.Tag != "" { desc, err = r.backend.ResolveTag(ctx, rreq.Repo, rreq.Tag) } else { - desc, err = r.backend.ResolveManifest(ctx, rreq.Repo, oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + desc, err = r.backend.ResolveManifest(ctx, rreq.Repo, digest) } if err != nil { return err @@ -152,7 +180,7 @@ func (r *registry) handleManifestHead(ctx context.Context, resp http.ResponseWri // to expect that the digest header is set on the response // even though the spec says it's only optional in this case. // TODO raise an issue on the spec about this. - resp.Header().Set("Docker-Content-Digest", string(desc.Digest)) + resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) } resp.Header().Set("Content-Type", desc.MediaType) resp.Header().Set("Content-Length", fmt.Sprint(desc.Size)) diff --git a/ociserver/registry.go b/ociserver/registry.go index df05803..8e48cc7 100644 --- a/ociserver/registry.go +++ b/ociserver/registry.go @@ -28,16 +28,11 @@ import ( "github.com/docker/oci" "github.com/docker/oci/internal/ocirequest" - ocispecroot "github.com/opencontainers/image-spec/specs-go" ) // debug causes debug messages to be emitted when running the server. const debug = false -var v2 = ocispecroot.Versioned{ - SchemaVersion: 2, -} - // Options holds options for the server. type Options struct { // WriteError is used to write error responses. It is passed the @@ -233,6 +228,6 @@ func (r *registry) setLocationHeader(resp http.ResponseWriter, isManifest bool, } } resp.Header().Set("Location", loc) - resp.Header().Set("Docker-Content-Digest", string(desc.Digest)) + resp.Header().Set("Docker-Content-Digest", desc.Digest.String()) return nil } diff --git a/ociserver/registry_test.go b/ociserver/registry_test.go index 2ecf1cf..6cfe07a 100644 --- a/ociserver/registry_test.go +++ b/ociserver/registry_test.go @@ -22,9 +22,9 @@ import ( "strings" "testing" + "github.com/docker/oci/ocidigest" "github.com/docker/oci/ocimem" "github.com/docker/oci/ociserver" - "github.com/opencontainers/go-digest" "github.com/stretchr/testify/require" ) @@ -674,5 +674,5 @@ func TestCalls(t *testing.T) { } func digestOf(s string) string { - return string(digest.FromString(s)) + return ocidigest.FromBytes([]byte(s)).String() } diff --git a/ociserver/writer.go b/ociserver/writer.go index 9b3a82f..4d86052 100644 --- a/ociserver/writer.go +++ b/ociserver/writer.go @@ -23,10 +23,9 @@ import ( "strconv" "github.com/docker/oci" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/docker/oci/internal/ocirequest" + "github.com/docker/oci/ocidigest" ) func (r *registry) handleBlobUploadBlob(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { @@ -36,15 +35,22 @@ func (r *registry) handleBlobUploadBlob(ctx context.Context, resp http.ResponseW // TODO check that Content-Type is application/octet-stream? mediaType := mediaTypeOctetStream + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + if digest.IsZero() { + return oci.NewError("badly formed digest", oci.ErrDigestInvalid.Code(), nil) + } desc, err := r.backend.PushBlob(req.Context(), rreq.Repo, oci.Descriptor{ MediaType: mediaType, Size: req.ContentLength, - Digest: oci.Digest(rreq.Digest), + Digest: digest, }, req.Body) if err != nil { return err } - if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/blobs/"+string(desc.Digest)); err != nil { + if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/blobs/"+desc.Digest.String()); err != nil { return err } resp.WriteHeader(http.StatusCreated) @@ -140,11 +146,18 @@ func (r *registry) handleBlobCompleteUpload(ctx context.Context, resp http.Respo if _, err := io.Copy(w, req.Body); err != nil { return fmt.Errorf("failed to copy data to %T: %v", w, err) } - desc, err := w.Commit(oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + if digest.IsZero() { + return oci.NewError("badly formed digest", oci.ErrDigestInvalid.Code(), nil) + } + desc, err := w.Commit(digest) if err != nil { return err } - if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/blobs/"+string(desc.Digest)); err != nil { + if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/blobs/"+desc.Digest.String()); err != nil { return err } resp.WriteHeader(http.StatusCreated) @@ -152,7 +165,14 @@ func (r *registry) handleBlobCompleteUpload(ctx context.Context, resp http.Respo } func (r *registry) handleBlobMount(ctx context.Context, resp http.ResponseWriter, req *http.Request, rreq *ocirequest.Request) error { - desc, err := r.backend.MountBlob(ctx, rreq.FromRepo, rreq.Repo, oci.Digest(rreq.Digest)) + digest, err := ocidigest.Parse(rreq.Digest) + if err != nil { + return err + } + if digest.IsZero() { + return oci.NewError("badly formed digest", oci.ErrDigestInvalid.Code(), nil) + } + desc, err := r.backend.MountBlob(ctx, rreq.FromRepo, rreq.Repo, digest) if err != nil { return err } @@ -174,12 +194,12 @@ func (r *registry) handleManifestPut(ctx context.Context, resp http.ResponseWrit if err != nil { return fmt.Errorf("cannot read content: %v", err) } - dig := digest.FromBytes(data) + dig := ocidigest.FromBytes(data) params := &oci.PushManifestParameters{} if rreq.Tag != "" { params.Tags = []string{rreq.Tag} } else { - if oci.Digest(rreq.Digest) != dig { + if rreq.Digest != dig.String() { return oci.ErrDigestInvalid } params.Tags = rreq.Tags @@ -192,11 +212,11 @@ func (r *registry) handleManifestPut(ctx context.Context, resp http.ResponseWrit if err != nil { return err } - if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/manifests/"+string(desc.Digest)); err != nil { + if err := r.setLocationHeader(resp, false, desc, "/v2/"+rreq.Repo+"/manifests/"+desc.Digest.String()); err != nil { return err } if subjectDesc != nil { - resp.Header().Set("OCI-Subject", string(subjectDesc.Digest)) + resp.Header().Set("OCI-Subject", subjectDesc.Digest.String()) } // TODO OCI-Subject header? resp.WriteHeader(http.StatusCreated) @@ -205,19 +225,21 @@ func (r *registry) handleManifestPut(ctx context.Context, resp http.ResponseWrit func subjectFromManifest(contentType string, data []byte) (*oci.Descriptor, error) { switch contentType { - case ocispec.MediaTypeImageManifest, - ocispec.MediaTypeImageIndex: - break - // TODO other manifest media types. + case oci.MediaTypeImageManifest, oci.MediaTypeDockerManifest, + oci.MediaTypeImageIndex, oci.MediaTypeDockerManifestList: default: return nil, nil } - var m struct { - Subject *oci.Descriptor `json:"subject"` - } + var m oci.IndexOrManifest if err := json.Unmarshal(data, &m); err != nil { return nil, err } + if m.MediaType == "" { + m.MediaType = contentType + } + if err := m.Validate(); err != nil { + return nil, err + } return m.Subject, nil } diff --git a/ocitest/ocitest.go b/ocitest/ocitest.go index e076a59..ab839e1 100644 --- a/ocitest/ocitest.go +++ b/ocitest/ocitest.go @@ -29,7 +29,7 @@ import ( "testing" "github.com/docker/oci" - "github.com/opencontainers/go-digest" + "github.com/docker/oci/ocidigest" "github.com/stretchr/testify/require" ) @@ -65,7 +65,7 @@ type RegistryContent map[string]RepoContent type RepoContent struct { // Manifests maps from manifest identifier to the contents of the manifest. // TODO support manifest indexes too. - Manifests map[string]oci.Manifest + Manifests map[string]oci.IndexOrManifest // Blobs maps from blob identifer to the contents of the blob. Blobs map[string]string @@ -90,6 +90,12 @@ type PushedRepoContent struct { Blobs map[string]oci.Descriptor } +// DigestRef returns a deterministic digest used as a symbolic content +// reference in RepoContent manifests before PushContent resolves descriptors. +func DigestRef(id string) oci.Digest { + return ocidigest.FromBytes([]byte("ocitest-ref:" + id)) +} + // PushContent pushes all the content in rc to r. // // It returns a map mapping repository name to the descriptors @@ -117,12 +123,17 @@ func PushRepoContent(r oci.Interface, repo string, repoc RepoContent) (PushedRep for id, blob := range repoc.Blobs { prc.Blobs[id] = oci.Descriptor{ - Digest: digest.FromString(blob), + Digest: ocidigest.FromBytes([]byte(blob)), Size: int64(len(blob)), MediaType: "application/binary", } } - manifests, manifestSeq, err := completedManifests(repoc, prc.Blobs) + blobRefs := make(map[string]oci.Descriptor) + for id, desc := range prc.Blobs { + blobRefs[id] = desc + blobRefs[DigestRef(id).String()] = desc + } + manifests, manifestSeq, err := completedManifests(repoc, blobRefs) if err != nil { return PushedRepoContent{}, err } @@ -181,6 +192,7 @@ type manifestContent struct { // for pushing to a registry in bottom-up order. func completedManifests(repoc RepoContent, blobs map[string]oci.Descriptor) (map[string]manifestContent, []manifestContent, error) { manifests := make(map[string]manifestContent) + manifestRefs := make(map[string]manifestContent) manifestSeq := make([]manifestContent, 0, len(repoc.Manifests)) // subject relationships can be arbitrarily deep, so continue iterating until // all the levels are completed. If at any point we can't make progress, we @@ -191,8 +203,8 @@ func completedManifests(repoc RepoContent, blobs map[string]oci.Descriptor) (map needMore := false need := func(digest oci.Digest) { needMore = true - if !required[string(digest)] { - required[string(digest)] = true + if !required[digest.String()] { + required[digest.String()] = true madeProgress = true } } @@ -202,7 +214,10 @@ func completedManifests(repoc RepoContent, blobs map[string]oci.Descriptor) (map } m1 := m if m1.Subject != nil { - mc, ok := manifests[string(m1.Subject.Digest)] + mc, ok := manifests[m1.Subject.Digest.String()] + if !ok { + mc, ok = manifestRefs[m1.Subject.Digest.String()] + } if !ok { need(m1.Subject.Digest) continue @@ -220,12 +235,13 @@ func completedManifests(repoc RepoContent, blobs map[string]oci.Descriptor) (map id: id, data: data, desc: oci.Descriptor{ - Digest: digest.FromBytes(data), + Digest: ocidigest.FromBytes(data), Size: int64(len(data)), MediaType: m.MediaType, }, } manifests[id] = mc + manifestRefs[DigestRef(id).String()] = mc madeProgress = true manifestSeq = append(manifestSeq, mc) } @@ -237,14 +253,20 @@ func completedManifests(repoc RepoContent, blobs map[string]oci.Descriptor) (map if _, ok := manifests[m]; ok { delete(required, m) } + if _, ok := manifestRefs[m]; ok { + delete(required, m) + } } return nil, nil, fmt.Errorf("no manifest found for ids %s", strings.Join(mapKeys(required), ", ")) } } } -func fillManifestDescriptors(m oci.Manifest, blobs map[string]oci.Descriptor) oci.Manifest { - m.Config = fillBlobDescriptor(m.Config, blobs) +func fillManifestDescriptors(m oci.IndexOrManifest, blobs map[string]oci.Descriptor) oci.IndexOrManifest { + if m.Config != nil { + config := fillBlobDescriptor(*m.Config, blobs) + m.Config = &config + } m.Layers = slices.Clone(m.Layers) for i, desc := range m.Layers { m.Layers[i] = fillBlobDescriptor(desc, blobs) @@ -253,7 +275,7 @@ func fillManifestDescriptors(m oci.Manifest, blobs map[string]oci.Descriptor) oc } func fillBlobDescriptor(d oci.Descriptor, blobs map[string]oci.Descriptor) oci.Descriptor { - blobDesc, ok := blobs[string(d.Digest)] + blobDesc, ok := blobs[d.Digest.String()] if !ok { panic(fmt.Errorf("no blob found with id %q", d.Digest)) } @@ -268,7 +290,7 @@ func fillBlobDescriptor(d oci.Descriptor, blobs map[string]oci.Descriptor) oci.D // MustPushBlob pushes a blob to the given repository and fails the test if it encounters an error. func (r Registry) MustPushBlob(repo string, data []byte) oci.Descriptor { desc := oci.Descriptor{ - Digest: digest.FromBytes(data), + Digest: ocidigest.FromBytes(data), Size: int64(len(data)), MediaType: "application/octet-stream", } @@ -289,7 +311,7 @@ func (r Registry) MustPushManifest(repo string, jsonObject any, tag string) ([]b require.NoError(r.T, err) require.NotEmpty(r.T, mt.MediaType) desc := oci.Descriptor{ - Digest: digest.FromBytes(data), + Digest: ocidigest.FromBytes(data), Size: int64(len(data)), MediaType: mt.MediaType, } @@ -327,7 +349,7 @@ func AssertBlobContent(t *testing.T, r oci.BlobReader, wantData []byte, wantMedi gotData, err := io.ReadAll(r) require.NoError(t, err, "error reading data") require.Equal(t, int64(len(wantData)), desc.Size, "mismatched content length") - require.Equal(t, digest.FromBytes(wantData), desc.Digest, "mismatched digest") + require.Equal(t, ocidigest.FromBytes(wantData), desc.Digest, "mismatched digest") require.Equal(t, wantData, gotData, "mismatched content") require.Equal(t, wantMediaType, desc.MediaType, "media type mismatch") } diff --git a/ociunify/lister.go b/ociunify/lister.go index 4b5f3dc..b071480 100644 --- a/ociunify/lister.go +++ b/ociunify/lister.go @@ -54,7 +54,7 @@ func (u unifier) Referrers(ctx context.Context, repo string, digest oci.Digest, } func compareDescriptor(d0, d1 oci.Descriptor) int { - return strings.Compare(string(d0.Digest), string(d1.Digest)) + return strings.Compare(d0.Digest.String(), d1.Digest.String()) } func mergeIter[T any](it0, it1 iter.Seq2[T, error], cmp func(T, T) int) iter.Seq2[T, error] {