Skip to content

Commit

Permalink
httptransport: accept OCI manifests for indexing
Browse files Browse the repository at this point in the history
Signed-off-by: Hank Donnay <hdonnay@redhat.com>
  • Loading branch information
hdonnay committed Oct 21, 2020
1 parent 37f7791 commit 1860b52
Show file tree
Hide file tree
Showing 4 changed files with 331 additions and 3 deletions.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ require (
github.com/jmoiron/sqlx v1.2.0
github.com/klauspost/compress v1.10.11
github.com/mattn/go-sqlite3 v1.11.0 // indirect
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1
github.com/prometheus/client_golang v0.9.4 // indirect
github.com/prometheus/procfs v0.0.8 // indirect
github.com/quay/claircore v0.1.13
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,9 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y=
Expand Down
91 changes: 88 additions & 3 deletions httptransport/indexhandler.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package httptransport

import (
"context"
"encoding/json"
"fmt"
"net/http"
"path"
"strings"

oci "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/quay/claircore"
je "github.com/quay/claircore/pkg/jsonerr"

Expand Down Expand Up @@ -41,8 +44,8 @@ func IndexHandler(serv indexer.StateIndexer) http.HandlerFunc {
return
}

var m claircore.Manifest
if err := json.NewDecoder(r.Body).Decode(&m); err != nil {
m, err := decodeManifest(ctx, r)
if err != nil {
resp := &je.Response{
Code: "bad-request",
Message: fmt.Sprintf("failed to deserialize manifest: %v", err),
Expand Down Expand Up @@ -70,7 +73,7 @@ func IndexHandler(serv indexer.StateIndexer) http.HandlerFunc {

// TODO Do we need some sort of background context embedded in the HTTP
// struct?
report, err := serv.Index(ctx, &m)
report, err := serv.Index(ctx, m)
if err != nil {
resp := &je.Response{
Code: "index-error",
Expand All @@ -88,3 +91,85 @@ func IndexHandler(serv indexer.StateIndexer) http.HandlerFunc {
err = json.NewEncoder(w).Encode(report)
}
}

const (
// Known manifest types we ingest.
typeOCIManifest = oci.MediaTypeImageManifest
typeNativeManifest = `application/vnd.projectquay.clair.mainfest.v1+json`
)

// DecodeManifest switches on the Request's Content-Type to consume the body.
//
// Defaults to expecting a native ClairCore Manifest.
func decodeManifest(ctx context.Context, r *http.Request) (*claircore.Manifest, error) {
defer r.Body.Close()
var m claircore.Manifest

t := r.Header.Get("content-type")
if i := strings.IndexByte(t, ';'); i != -1 {
t = strings.TrimSpace(t[:i])
}
switch t {
case typeOCIManifest:
var om oci.Manifest
if err := json.NewDecoder(r.Body).Decode(&om); err != nil {
return nil, err
}
if err := nativeFromOCI(&m, &om); err != nil {
return nil, err
}
case typeNativeManifest, "application/json", "":
if err := json.NewDecoder(r.Body).Decode(&m); err != nil {
return nil, err
}
default:
return nil, fmt.Errorf("unknown content-type %q", t)
}
return &m, nil
}

// These are the layer types we accept inside an OCI Manifest.
var ociLayerTypes = map[string]struct{}{
oci.MediaTypeImageLayer: {},
oci.MediaTypeImageLayerGzip: {},
oci.MediaTypeImageLayer + "+zstd": {}, // The specs package doesn't have zstd, oddly.
}

// NativeFromOCI populates the Manifest from the OCI Manifest, reporting an
// error if something is invalid.
func nativeFromOCI(m *claircore.Manifest, o *oci.Manifest) error {
const header = `header:`
var err error

m.Hash, err = claircore.ParseDigest(o.Config.Digest.String())
if err != nil {
return fmt.Errorf("unable to parse manifest digest %q: %w", o.Config.Digest, err)
}

for _, u := range o.Layers {
if len(u.URLs) == 0 {
// Manifest is missing URLs.
// They're optional in the spec, but we need them for obvious reasons.
return fmt.Errorf("missing URLs for layer %q", u.Digest)
}
if _, ok := ociLayerTypes[u.MediaType]; !ok {
return fmt.Errorf("invalid media type for layer %q", u.Digest)
}
l := claircore.Layer{
URI: u.URLs[0],
}
l.Hash, err = claircore.ParseDigest(u.Digest.String())
if err != nil {
return fmt.Errorf("unable to parse layer digest %q: %w", u.Digest, err)
}
for k, v := range u.Annotations {
if !strings.HasPrefix(k, header) {
continue
}
l.Headers[strings.TrimPrefix(k, header)] = []string{v}
}
m.Layers = append(m.Layers, &l)
}

return nil
}
239 changes: 239 additions & 0 deletions httptransport/indexhandler_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
package httptransport

import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/opencontainers/go-digest"
oci "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/quay/claircore"
)

func TestNativeFromOCI(t *testing.T) {
t.Parallel()

var cmpOpts = cmp.Options{
cmp.Comparer(func(a, b claircore.Digest) bool { return a.String() == b.String() }),
cmpopts.IgnoreUnexported(claircore.Layer{}),
}
type testcase struct {
Name string
Want claircore.Manifest
In oci.Manifest
Err bool
}
Run := func(tc *testcase) func(*testing.T) {
return func(t *testing.T) {
var got claircore.Manifest

err := nativeFromOCI(&got, &tc.In)
if (err != nil) != tc.Err {
t.Errorf("unexpected error: %v", err)
}

if got, want := &got, &tc.Want; !cmp.Equal(got, want, cmpOpts) {
t.Error(cmp.Diff(got, want, cmpOpts))
}
}
}

tt := []testcase{
{
Name: "EmptyDigest",
In: oci.Manifest{},
Err: true,
},
{
Name: "BadDigest",
In: oci.Manifest{
Config: oci.Descriptor{
Digest: digest.Digest("xxx:yyy"),
},
},
Err: true,
},
{
Name: "BadURLs",
In: oci.Manifest{
Config: oci.Descriptor{
Digest: digest.FromString("good manifest"),
},
Layers: []oci.Descriptor{
{URLs: nil},
},
},
Want: claircore.Manifest{
Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"),
},
Err: true,
},
{
Name: "BadMediaType",
In: oci.Manifest{
Config: oci.Descriptor{
Digest: digest.FromString("good manifest"),
},
Layers: []oci.Descriptor{
{
MediaType: `fake/media-type`,
URLs: []string{"http://localhost/real/layer"},
},
},
},
Want: claircore.Manifest{
Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"),
},
Err: true,
},
{
Name: "BadLayerDigest",
In: oci.Manifest{
Config: oci.Descriptor{
Digest: digest.FromString("good manifest"),
},
Layers: []oci.Descriptor{
{
Digest: digest.Digest("xxx:yyy"),
MediaType: oci.MediaTypeImageLayer,
URLs: []string{"http://localhost/real/layer"},
},
},
},
Want: claircore.Manifest{
Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"),
},
Err: true,
},
{
Name: "OK",
Want: claircore.Manifest{
Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"),
Layers: []*claircore.Layer{
{
Hash: claircore.MustParseDigest("sha256:ba54d2c66022c637137ad0896ba5fb790847755be51b08bc472ffab5fdd76b1b"),
URI: "http://localhost/real/layer",
},
},
},
In: oci.Manifest{
Config: oci.Descriptor{
Digest: digest.FromString("good manifest"),
},
Layers: []oci.Descriptor{
{
Digest: digest.FromString("cool layer"),
MediaType: oci.MediaTypeImageLayer,
URLs: []string{"http://localhost/real/layer"},
},
},
},
},
}

for _, tc := range tt {
t.Run(tc.Name, Run(&tc))
}
}

func TestDecodeManifest(t *testing.T) {
t.Parallel()
ctx := context.Background()

type testcase struct {
Name string
Want claircore.Manifest
In *http.Request
Err bool
}
Run := func(tc *testcase) func(*testing.T) {
return func(t *testing.T) {
got, err := decodeManifest(ctx, tc.In)
if err != nil {
t.Log(err)
}
if (err != nil) != tc.Err {
t.Errorf("unexpected error: %v", err)
}
_ = got
}
}

const (
goodOCI = `{
"mediaType":"` + oci.MediaTypeImageManifest + `",
"config":{"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"},
"layers":[{
"mediaType":"` + oci.MediaTypeImageLayer + `",
"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19",
"urls":["http://example.com/layer"]
}]}`
errorOCI = `{
"mediaType":"` + oci.MediaTypeImageManifest + `",
"config":{"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"},
"layers":[{
"mediaType":"` + oci.MediaTypeImageLayer + `",
"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"
}]}`
)
tt := []testcase{
{
Name: "NoHeaders",
In: httptest.NewRequest("", "/", strings.NewReader(`{}`)),
},
{
Name: "BadContentType",
In: httptest.NewRequest("", "/", strings.NewReader(`{}`)),
Err: true,
},
{
Name: "Default",
In: httptest.NewRequest("", "/", strings.NewReader(`{}`)),
},
{
Name: "Default+Error",
In: httptest.NewRequest("", "/", strings.NewReader(`""`)),
Err: true,
},
{
Name: "Claircore",
In: httptest.NewRequest("", "/", strings.NewReader(`{}`)),
},
{
Name: "OCIManifest",
In: httptest.NewRequest("", "/", strings.NewReader(goodOCI)),
},
{
Name: "OCIManifest+DecodeError",
In: httptest.NewRequest("", "/", strings.NewReader(`""`)),
Err: true,
},
{
Name: "OCIManifest+TranslateError",
In: httptest.NewRequest("", "/", strings.NewReader(errorOCI)),
Err: true,
},
}
// Adjust headers
for _, tc := range tt {
switch tc.Name {
case "NoHeaders":
case "BadContentType":
tc.In.Header.Set(`content-type`, `text/plain; charset=UTF-8`)
case "OCIManifest", "OCIManifest+DecodeError", "OCIManifest+TranslateError":
tc.In.Header.Set(`content-type`, oci.MediaTypeImageManifest)
case "Claircore":
tc.In.Header.Set(`content-type`, `application/json; charset=UTF-8`)
default:
tc.In.Header.Set(`content-type`, `application/json; charset=UTF-8`)
}
}

for _, tc := range tt {
t.Run(tc.Name, Run(&tc))
}
}

0 comments on commit 1860b52

Please sign in to comment.