From 1860b52cbfc509d3a8e70caf3c08f1824bdf722c Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Mon, 12 Oct 2020 12:51:25 -0500 Subject: [PATCH] httptransport: accept OCI manifests for indexing Signed-off-by: Hank Donnay --- go.mod | 2 + go.sum | 2 + httptransport/indexhandler.go | 91 ++++++++++- httptransport/indexhandler_test.go | 239 +++++++++++++++++++++++++++++ 4 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 httptransport/indexhandler_test.go diff --git a/go.mod b/go.mod index 75f466d859..a2e504d343 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,8 @@ require ( github.com/jmoiron/sqlx v1.2.0 github.com/klauspost/compress v1.10.11 github.com/mattn/go-sqlite3 v1.11.0 // indirect + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.0.1 github.com/prometheus/client_golang v0.9.4 // indirect github.com/prometheus/procfs v0.0.8 // indirect github.com/quay/claircore v0.1.13 diff --git a/go.sum b/go.sum index 4bff6c6b7e..7f861db0a8 100644 --- a/go.sum +++ b/go.sum @@ -471,7 +471,9 @@ github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= diff --git a/httptransport/indexhandler.go b/httptransport/indexhandler.go index b06d192759..e0a19614dc 100644 --- a/httptransport/indexhandler.go +++ b/httptransport/indexhandler.go @@ -1,11 +1,14 @@ package httptransport import ( + "context" "encoding/json" "fmt" "net/http" "path" + "strings" + oci "github.com/opencontainers/image-spec/specs-go/v1" "github.com/quay/claircore" je "github.com/quay/claircore/pkg/jsonerr" @@ -41,8 +44,8 @@ func IndexHandler(serv indexer.StateIndexer) http.HandlerFunc { return } - var m claircore.Manifest - if err := json.NewDecoder(r.Body).Decode(&m); err != nil { + m, err := decodeManifest(ctx, r) + if err != nil { resp := &je.Response{ Code: "bad-request", Message: fmt.Sprintf("failed to deserialize manifest: %v", err), @@ -70,7 +73,7 @@ func IndexHandler(serv indexer.StateIndexer) http.HandlerFunc { // TODO Do we need some sort of background context embedded in the HTTP // struct? - report, err := serv.Index(ctx, &m) + report, err := serv.Index(ctx, m) if err != nil { resp := &je.Response{ Code: "index-error", @@ -88,3 +91,85 @@ func IndexHandler(serv indexer.StateIndexer) http.HandlerFunc { err = json.NewEncoder(w).Encode(report) } } + +const ( + // Known manifest types we ingest. + typeOCIManifest = oci.MediaTypeImageManifest + typeNativeManifest = `application/vnd.projectquay.clair.mainfest.v1+json` +) + +// DecodeManifest switches on the Request's Content-Type to consume the body. +// +// Defaults to expecting a native ClairCore Manifest. +func decodeManifest(ctx context.Context, r *http.Request) (*claircore.Manifest, error) { + defer r.Body.Close() + var m claircore.Manifest + + t := r.Header.Get("content-type") + if i := strings.IndexByte(t, ';'); i != -1 { + t = strings.TrimSpace(t[:i]) + } + switch t { + case typeOCIManifest: + var om oci.Manifest + if err := json.NewDecoder(r.Body).Decode(&om); err != nil { + return nil, err + } + if err := nativeFromOCI(&m, &om); err != nil { + return nil, err + } + case typeNativeManifest, "application/json", "": + if err := json.NewDecoder(r.Body).Decode(&m); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("unknown content-type %q", t) + } + return &m, nil +} + +// These are the layer types we accept inside an OCI Manifest. +var ociLayerTypes = map[string]struct{}{ + oci.MediaTypeImageLayer: {}, + oci.MediaTypeImageLayerGzip: {}, + oci.MediaTypeImageLayer + "+zstd": {}, // The specs package doesn't have zstd, oddly. +} + +// NativeFromOCI populates the Manifest from the OCI Manifest, reporting an +// error if something is invalid. +func nativeFromOCI(m *claircore.Manifest, o *oci.Manifest) error { + const header = `header:` + var err error + + m.Hash, err = claircore.ParseDigest(o.Config.Digest.String()) + if err != nil { + return fmt.Errorf("unable to parse manifest digest %q: %w", o.Config.Digest, err) + } + + for _, u := range o.Layers { + if len(u.URLs) == 0 { + // Manifest is missing URLs. + // They're optional in the spec, but we need them for obvious reasons. + return fmt.Errorf("missing URLs for layer %q", u.Digest) + } + if _, ok := ociLayerTypes[u.MediaType]; !ok { + return fmt.Errorf("invalid media type for layer %q", u.Digest) + } + l := claircore.Layer{ + URI: u.URLs[0], + } + l.Hash, err = claircore.ParseDigest(u.Digest.String()) + if err != nil { + return fmt.Errorf("unable to parse layer digest %q: %w", u.Digest, err) + } + for k, v := range u.Annotations { + if !strings.HasPrefix(k, header) { + continue + } + l.Headers[strings.TrimPrefix(k, header)] = []string{v} + } + m.Layers = append(m.Layers, &l) + } + + return nil +} diff --git a/httptransport/indexhandler_test.go b/httptransport/indexhandler_test.go new file mode 100644 index 0000000000..63e78c6d7a --- /dev/null +++ b/httptransport/indexhandler_test.go @@ -0,0 +1,239 @@ +package httptransport + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/quay/claircore" +) + +func TestNativeFromOCI(t *testing.T) { + t.Parallel() + + var cmpOpts = cmp.Options{ + cmp.Comparer(func(a, b claircore.Digest) bool { return a.String() == b.String() }), + cmpopts.IgnoreUnexported(claircore.Layer{}), + } + type testcase struct { + Name string + Want claircore.Manifest + In oci.Manifest + Err bool + } + Run := func(tc *testcase) func(*testing.T) { + return func(t *testing.T) { + var got claircore.Manifest + + err := nativeFromOCI(&got, &tc.In) + if (err != nil) != tc.Err { + t.Errorf("unexpected error: %v", err) + } + + if got, want := &got, &tc.Want; !cmp.Equal(got, want, cmpOpts) { + t.Error(cmp.Diff(got, want, cmpOpts)) + } + } + } + + tt := []testcase{ + { + Name: "EmptyDigest", + In: oci.Manifest{}, + Err: true, + }, + { + Name: "BadDigest", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.Digest("xxx:yyy"), + }, + }, + Err: true, + }, + { + Name: "BadURLs", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + {URLs: nil}, + }, + }, + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + }, + Err: true, + }, + { + Name: "BadMediaType", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + { + MediaType: `fake/media-type`, + URLs: []string{"http://localhost/real/layer"}, + }, + }, + }, + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + }, + Err: true, + }, + { + Name: "BadLayerDigest", + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + { + Digest: digest.Digest("xxx:yyy"), + MediaType: oci.MediaTypeImageLayer, + URLs: []string{"http://localhost/real/layer"}, + }, + }, + }, + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + }, + Err: true, + }, + { + Name: "OK", + Want: claircore.Manifest{ + Hash: claircore.MustParseDigest("sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"), + Layers: []*claircore.Layer{ + { + Hash: claircore.MustParseDigest("sha256:ba54d2c66022c637137ad0896ba5fb790847755be51b08bc472ffab5fdd76b1b"), + URI: "http://localhost/real/layer", + }, + }, + }, + In: oci.Manifest{ + Config: oci.Descriptor{ + Digest: digest.FromString("good manifest"), + }, + Layers: []oci.Descriptor{ + { + Digest: digest.FromString("cool layer"), + MediaType: oci.MediaTypeImageLayer, + URLs: []string{"http://localhost/real/layer"}, + }, + }, + }, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, Run(&tc)) + } +} + +func TestDecodeManifest(t *testing.T) { + t.Parallel() + ctx := context.Background() + + type testcase struct { + Name string + Want claircore.Manifest + In *http.Request + Err bool + } + Run := func(tc *testcase) func(*testing.T) { + return func(t *testing.T) { + got, err := decodeManifest(ctx, tc.In) + if err != nil { + t.Log(err) + } + if (err != nil) != tc.Err { + t.Errorf("unexpected error: %v", err) + } + _ = got + } + } + + const ( + goodOCI = `{ + "mediaType":"` + oci.MediaTypeImageManifest + `", + "config":{"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"}, + "layers":[{ + "mediaType":"` + oci.MediaTypeImageLayer + `", + "digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19", + "urls":["http://example.com/layer"] + }]}` + errorOCI = `{ + "mediaType":"` + oci.MediaTypeImageManifest + `", + "config":{"digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19"}, + "layers":[{ + "mediaType":"` + oci.MediaTypeImageLayer + `", + "digest":"sha256:a3114909b7f9d6c4a680e04c3f6eacaeb80c4d9d8d802f81e9b9cf0a29d26e19" + }]}` + ) + tt := []testcase{ + { + Name: "NoHeaders", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + }, + { + Name: "BadContentType", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + Err: true, + }, + { + Name: "Default", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + }, + { + Name: "Default+Error", + In: httptest.NewRequest("", "/", strings.NewReader(`""`)), + Err: true, + }, + { + Name: "Claircore", + In: httptest.NewRequest("", "/", strings.NewReader(`{}`)), + }, + { + Name: "OCIManifest", + In: httptest.NewRequest("", "/", strings.NewReader(goodOCI)), + }, + { + Name: "OCIManifest+DecodeError", + In: httptest.NewRequest("", "/", strings.NewReader(`""`)), + Err: true, + }, + { + Name: "OCIManifest+TranslateError", + In: httptest.NewRequest("", "/", strings.NewReader(errorOCI)), + Err: true, + }, + } + // Adjust headers + for _, tc := range tt { + switch tc.Name { + case "NoHeaders": + case "BadContentType": + tc.In.Header.Set(`content-type`, `text/plain; charset=UTF-8`) + case "OCIManifest", "OCIManifest+DecodeError", "OCIManifest+TranslateError": + tc.In.Header.Set(`content-type`, oci.MediaTypeImageManifest) + case "Claircore": + tc.In.Header.Set(`content-type`, `application/json; charset=UTF-8`) + default: + tc.In.Header.Set(`content-type`, `application/json; charset=UTF-8`) + } + } + + for _, tc := range tt { + t.Run(tc.Name, Run(&tc)) + } +}