From 0282f68bf381a5b0a592079819e38b3d88296f92 Mon Sep 17 00:00:00 2001 From: Hank Donnay Date: Tue, 17 Mar 2020 16:22:41 -0400 Subject: [PATCH] clairctl: report command Signed-off-by: Hank Donnay --- cmd/clairctl/client.go | 231 ++++++++++++++++++++++++++++++++++ cmd/clairctl/jsonformatter.go | 19 +++ cmd/clairctl/main.go | 1 + cmd/clairctl/manifest.go | 25 ++-- cmd/clairctl/report.go | 205 ++++++++++++++++++++++++++++++ cmd/clairctl/textformatter.go | 60 +++++++++ cmd/clairctl/xmlformatter.go | 116 +++++++++++++++++ go.mod | 1 + go.sum | 2 + 9 files changed, 643 insertions(+), 17 deletions(-) create mode 100644 cmd/clairctl/client.go create mode 100644 cmd/clairctl/jsonformatter.go create mode 100644 cmd/clairctl/report.go create mode 100644 cmd/clairctl/textformatter.go create mode 100644 cmd/clairctl/xmlformatter.go diff --git a/cmd/clairctl/client.go b/cmd/clairctl/client.go new file mode 100644 index 0000000000..2d7ec100ed --- /dev/null +++ b/cmd/clairctl/client.go @@ -0,0 +1,231 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "path" + "sync" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + "github.com/quay/claircore" + "github.com/tomnomnom/linkheader" +) + +const ( + userAgent = `clairctl/1` +) + +var ( + rtMu sync.Mutex + rtMap = map[string]http.RoundTripper{} +) + +func rt(ref string) (http.RoundTripper, error) { + r, err := name.ParseReference(ref) + if err != nil { + return nil, err + } + repo := r.Context() + key := repo.String() + rtMu.Lock() + defer rtMu.Unlock() + if v, ok := rtMap[key]; ok { + return v, nil + } + + auth, err := authn.DefaultKeychain.Resolve(repo) + if err != nil { + return nil, err + } + rt, err := transport.New(repo.Registry, auth, http.DefaultTransport, []string{repo.Scope("pull")}) + if err != nil { + return nil, err + } + rtMap[key] = rt + return rt, nil +} + +// TODO Maybe turn this into a real client, once it's proved useful. +type Client struct { + api *url.URL + client *http.Client + + mu sync.RWMutex + validator map[string]string +} + +func NewClient(root string) (*Client, error) { + api, err := url.Parse(root) + if err != nil { + return nil, err + } + return &Client{ + api: api, + client: &http.Client{}, + validator: make(map[string]string), + }, nil +} + +func (c *Client) getValidator(path string) string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.validator[path] +} + +func (c *Client) setValidator(path, v string) { + debug.Printf("setting validator %q → %q", path, v) + c.mu.Lock() + defer c.mu.Unlock() + c.validator[path] = v +} + +var errNeedManifest = errors.New("manifest needed but not supplied") + +func (c *Client) IndexReport(ctx context.Context, id claircore.Digest, m *claircore.Manifest) error { + var ( + req *http.Request + res *http.Response + ) + fp, err := c.api.Parse(path.Join("index_report", id.String())) + if err != nil { + debug.Printf("unable to construct index_report url: %v", err) + return err + } + req = c.request(ctx, fp, http.MethodGet) + res, err = c.client.Do(req) + if res != nil { + // Don't actually care. + res.Body.Close() + } + if err != nil { + debug.Printf("request failed for url %q: %v", req.URL.String(), err) + return err + } + debug.Printf("%s %s: %s", res.Request.Method, res.Request.URL.Path, res.Status) + switch res.StatusCode { + case http.StatusOK, http.StatusNotFound: + debug.Printf("need to post manifest %v", id) + case http.StatusNotModified: + return nil + default: + return fmt.Errorf("unexpected return status: %d", res.StatusCode) + } + + if m == nil { + debug.Printf("don't have needed manifest %v", id) + return errNeedManifest + } + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(m); err != nil { + debug.Printf("unable to encode json payload: %v", err) + return err + } + ru, err := c.api.Parse("index_report") + if err != nil { + debug.Printf("unable to construct index_report url: %v", err) + return err + } + + req = c.request(ctx, ru, http.MethodPost) + req.Body = ioutil.NopCloser(&buf) + res, err = c.client.Do(req) + if res != nil { + defer res.Body.Close() + } + if err != nil { + debug.Printf("request failed for url %q: %v", req.URL.String(), err) + return err + } + debug.Printf("%s %s: %s", res.Request.Method, res.Request.URL.Path, res.Status) + switch res.StatusCode { + case http.StatusOK: + case http.StatusCreated: + // + default: + return fmt.Errorf("unexpected return status: %d", res.StatusCode) + } + var report claircore.IndexReport + if err := json.NewDecoder(res.Body).Decode(&report); err != nil { + debug.Printf("unable to decode json payload: %v", err) + return err + } + if !report.Success && report.Err != "" { + return errors.New("indexer error: " + report.Err) + } + if v := res.Header.Get("etag"); v != "" { + ls := linkheader.ParseMultiple(res.Header[http.CanonicalHeaderKey("link")]). + FilterByRel("https://projectquay.io/clair/v1/index_report") + if len(ls) > 0 { + u, err := url.Parse(ls[0].URL) + if err != nil { + return err + } + c.setValidator(u.Path, v) + } + } + return nil +} + +func (c *Client) VulnerabilityReport(ctx context.Context, id claircore.Digest) (*claircore.VulnerabilityReport, error) { + var ( + req *http.Request + res *http.Response + ) + u, err := c.api.Parse(path.Join("vulnerability_report", id.String())) + if err != nil { + debug.Printf("unable to construct vulnerability_report url: %v", err) + return nil, err + } + req = c.request(ctx, u, http.MethodGet) + res, err = c.client.Do(req) + if res != nil { + defer res.Body.Close() + } + if err != nil { + debug.Printf("request failed for url %q: %v", req.URL.String(), err) + return nil, err + } + debug.Printf("%s %s: %s", res.Request.Method, res.Request.URL.Path, res.Status) + switch res.StatusCode { + case http.StatusOK: + case http.StatusNotModified: + // ??? + return nil, errors.New("not modified") + default: + return nil, fmt.Errorf("unexpected return status: %d", res.StatusCode) + } + var report claircore.VulnerabilityReport + if err := json.NewDecoder(res.Body).Decode(&report); err != nil { + debug.Printf("unable to decode json payload: %v", err) + return nil, err + } + + return &report, nil +} + +func (c *Client) request(ctx context.Context, u *url.URL, m string) *http.Request { + req := &http.Request{ + Method: m, + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: make(http.Header), + Body: nil, + Host: u.Host, + } + req = req.WithContext(ctx) + req.Header.Set("user-agent", userAgent) + if v := c.getValidator(u.EscapedPath()); v != "" { + req.Header.Set("if-none-match", v) + } + return req +} diff --git a/cmd/clairctl/jsonformatter.go b/cmd/clairctl/jsonformatter.go new file mode 100644 index 0000000000..971bbdbf73 --- /dev/null +++ b/cmd/clairctl/jsonformatter.go @@ -0,0 +1,19 @@ +package main + +import ( + "encoding/json" + "io" +) + +var _ Formatter = (*jsonFormatter)(nil) + +// JsonFormatter is a very simple formatter; it just calls +// (*json.Encoder).Encode. +type jsonFormatter struct { + enc *json.Encoder + io.Closer +} + +func (f *jsonFormatter) Format(r *Result) error { + return f.enc.Encode(r.Report) +} diff --git a/cmd/clairctl/main.go b/cmd/clairctl/main.go index 9ab051af97..a56b019e8c 100644 --- a/cmd/clairctl/main.go +++ b/cmd/clairctl/main.go @@ -31,6 +31,7 @@ func main() { }, Commands: []*cli.Command{ ManifestCmd, + ReportCmd, }, Flags: []cli.Flag{ &cli.BoolFlag{ diff --git a/cmd/clairctl/manifest.go b/cmd/clairctl/manifest.go index e89ef882a0..321865f6e5 100644 --- a/cmd/clairctl/manifest.go +++ b/cmd/clairctl/manifest.go @@ -11,10 +11,8 @@ import ( "path" "strings" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/quay/claircore" "github.com/urfave/cli/v2" "golang.org/x/sync/errgroup" @@ -53,7 +51,7 @@ func manifestAction(c *cli.Context) error { eg.Go(func() error { m, err := Inspect(ctx, name) if err != nil { - debug.Printf("%s: err: %v", name) + debug.Printf("%s: err: %v", name, err) return err } debug.Printf("%s: ok", name) @@ -70,20 +68,15 @@ func manifestAction(c *cli.Context) error { } func Inspect(ctx context.Context, r string) (*claircore.Manifest, error) { - ref, err := name.ParseReference(r) - if err != nil { - return nil, err - } - repo := ref.Context() - auth, err := authn.DefaultKeychain.Resolve(repo) + rt, err := rt(r) if err != nil { return nil, err } - rt, err := transport.New(repo.Registry, auth, http.DefaultTransport, []string{repo.Scope("pull")}) + + ref, err := name.ParseReference(r) if err != nil { return nil, err } - desc, err := remote.Get(ref, remote.WithTransport(rt)) if err != nil { return nil, err @@ -92,18 +85,15 @@ func Inspect(ctx context.Context, r string) (*claircore.Manifest, error) { if err != nil { return nil, err } - - h, err := img.Digest() + dig, err := img.Digest() if err != nil { return nil, err } - ccd, err := claircore.ParseDigest(h.String()) + ccd, err := claircore.ParseDigest(dig.String()) if err != nil { return nil, err } - out := claircore.Manifest{ - Hash: ccd, - } + out := claircore.Manifest{Hash: ccd} debug.Printf("%s: found manifest %v", r, ccd) ls, err := img.Layers() @@ -112,6 +102,7 @@ func Inspect(ctx context.Context, r string) (*claircore.Manifest, error) { } debug.Printf("%s: found %d layers", r, len(ls)) + repo := ref.Context() rURL := url.URL{ Scheme: repo.Scheme(), Host: repo.RegistryStr(), diff --git a/cmd/clairctl/report.go b/cmd/clairctl/report.go new file mode 100644 index 0000000000..f98ed21dd1 --- /dev/null +++ b/cmd/clairctl/report.go @@ -0,0 +1,205 @@ +package main + +import ( + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "log" + "os" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/quay/claircore" + "github.com/urfave/cli/v2" + "golang.org/x/sync/errgroup" +) + +// ReportCmd is the "report" subcommand. +var ReportCmd = &cli.Command{ + Name: "report", + Description: "Request and print a Clair vulnerability report for the provided container(s).", + Action: reportAction, + Usage: "request vulnerability reports for the named containers", + ArgsUsage: "container...", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "api", + Usage: "URL for the clairv4 v1 API.", + Value: "http://localhost:6060/api/v1/", + }, + &cli.GenericFlag{ + Name: "out", + Aliases: []string{"o"}, + Usage: "output format: text, json, xml", + DefaultText: "text", + Value: &outFmt{}, + }, + }, +} + +// OutFmt is a flag that creates a Formatter for us. +type outFmt struct { + fmt string +} + +func (o *outFmt) Set(v string) error { + switch v { + case "text": + case "json": + case "xml": + default: + return fmt.Errorf("unrecognized output format %q", v) + } + o.fmt = v + return nil +} + +func (o *outFmt) String() string { + return o.fmt +} + +func (o *outFmt) Formatter(w io.WriteCloser) Formatter { + switch o.fmt { + case "", "text": + debug.Println("using text output") + r, err := newTextFormatter(w) + if err != nil { + panic(err) + } + return r + case "json": + debug.Println("using json output") + return &jsonFormatter{ + enc: json.NewEncoder(w), + Closer: w, + } + case "xml": + debug.Println("using xml output") + return &xmlFormatter{ + enc: xml.NewEncoder(w), + c: w, + } + default: + } + panic("unreachable") // Somehow dodged the initial Set call. +} + +// Formatter is the common interface for presenting results. +type Formatter interface { + Format(*Result) error + io.Closer +} + +// Result is the result of a Clair request flow. +// +// Users should examine Err first to determine if the request succeeded. +type Result struct { + Name string + Err error + Report *claircore.VulnerabilityReport +} + +func reportAction(c *cli.Context) error { + args := c.Args() + if args.Len() == 0 { + return errors.New("missing needed arguments") + } + + cc, err := NewClient(c.String("api")) + if err != nil { + return err + } + + result := make(chan *Result) + done := make(chan struct{}) + eg, ctx := errgroup.WithContext(c.Context) + go func() { + defer close(done) + out := c.Generic("out").(*outFmt) + f := out.Formatter(os.Stdout) + defer f.Close() + for r := range result { + if err := f.Format(r); err != nil { + log.Println(err) + } + } + }() + + for i := 0; i < args.Len(); i++ { + ref := args.Get(i) + debug.Printf("%s: fetching", ref) + eg.Go(func() error { + d, err := resolveRef(ref) + if err != nil { + debug.Printf("%s: error: %v", ref, err) + return err + } + debug.Printf("%s: manifest: %v", ref, d) + + // This bit is tricky: + // + // Initially start with a nil manifest, which optimistically + // prevents us from generating one. + // + // If we need the manifest, populate the manifest and jump to Again. + var m *claircore.Manifest + Again: + err = cc.IndexReport(ctx, d, m) + switch { + case err == nil: + case errors.Is(err, errNeedManifest): + m, err = Inspect(ctx, ref) + if err != nil { + debug.Printf("%s: manifest error: %v", ref, err) + return err + } + goto Again + default: + debug.Printf("%s: index error: %v", ref, err) + return err + } + + r := Result{ + Name: ref, + } + r.Report, r.Err = cc.VulnerabilityReport(ctx, d) + result <- &r + return nil + }) + } + if err := eg.Wait(); err != nil { + return err + } + close(result) + <-done + return nil + +} + +func resolveRef(r string) (claircore.Digest, error) { + var d claircore.Digest + rt, err := rt(r) + if err != nil { + return d, err + } + + ref, err := name.ParseReference(r) + if err != nil { + return d, err + } + desc, err := remote.Get(ref, remote.WithTransport(rt)) + if err != nil { + return d, err + } + img, err := desc.Image() + if err != nil { + return d, err + } + dig, err := img.Digest() + if err != nil { + return d, err + } + return claircore.ParseDigest(dig.String()) +} diff --git a/cmd/clairctl/textformatter.go b/cmd/clairctl/textformatter.go new file mode 100644 index 0000000000..2a69a4e00a --- /dev/null +++ b/cmd/clairctl/textformatter.go @@ -0,0 +1,60 @@ +package main + +import ( + "io" + "path" + "text/tabwriter" + "text/template" +) + +var _ Formatter = (*textFormatter)(nil) + +// TextFormatter uses a custom text template and a tabwriter to present columnar +// output. +type textFormatter struct { + tmpl *template.Template + w *tabwriter.Writer + io.Closer +} + +var funcmap = template.FuncMap{ + "base": path.Base, +} + +func newTextFormatter(w io.WriteCloser) (*textFormatter, error) { + tmpl, err := template.New("report").Funcs(funcmap).Parse(tabwriterTmpl) + if err != nil { + return nil, err + } + tw := tabwriter.NewWriter(w, 0, 0, 1, ' ', 0) + r := textFormatter{ + tmpl: tmpl, + w: tw, + Closer: w, + } + return &r, nil +} + +const tabwriterTmpl = ` +{{- define "ok" -}} +{{base .Name}} ok +{{end}} +{{- define "err" -}} +{{base .Name}} error {{.Err}} +{{end}} +{{- define "found" -}} +{{with $r := .}}{{range $id, $v := .Report.PackageVulnerabilities}}{{range $d := $v -}} +{{base $r.Name}} found {{with index $r.Report.Packages $id}}{{.Name}} {{.Version}}{{end}} + {{- with index $r.Report.Vulnerabilities $d}} {{.Name}} + {{- with .FixedInVersion}} (fixed: {{.}}){{end}}{{end}} +{{end}}{{end}}{{end}}{{end}} +{{- /* The following is the actual bit of the template that runs per item. */ -}} +{{if .Err}}{{template "err" .}} +{{- else if ne (len .Report.PackageVulnerabilities) 0}}{{template "found" .}} +{{- else}}{{template "ok" .}} +{{- end}}` + +func (f *textFormatter) Format(r *Result) error { + defer f.w.Flush() + return f.tmpl.Execute(f.w, r) +} diff --git a/cmd/clairctl/xmlformatter.go b/cmd/clairctl/xmlformatter.go new file mode 100644 index 0000000000..2f303c8add --- /dev/null +++ b/cmd/clairctl/xmlformatter.go @@ -0,0 +1,116 @@ +package main + +import ( + "encoding/xml" + "fmt" + "io" + "strings" + "sync" +) + +var _ Formatter = (*xmlFormatter)(nil) + +// XmlFormatter is an attempt to create jUnit-compatible output. +type xmlFormatter struct { + sync.Mutex + enc *xml.Encoder + c io.Closer + out junitOut +} + +func (f *xmlFormatter) Reset(w io.WriteCloser) error { + f.c = w + io.WriteString(w, xml.Header) + f.enc = xml.NewEncoder(w) + return nil +} + +func (f *xmlFormatter) Format(r *Result) error { + var j junitTestsuite + if err := j.Init(r); err != nil { + return err + } + f.Lock() + defer f.Unlock() + f.out.Tests = append(f.out.Tests, &j) + return nil +} + +func (f *xmlFormatter) Close() error { + defer f.c.Close() + return f.enc.Encode(f.out) +} + +type junitOut struct { + XMLName xml.Name `xml:"testsuites"` + Tests []*junitTestsuite +} + +type junitTestsuite struct { + XMLName xml.Name `xml:"testsuite"` + Name string `xml:"name,attr"` + NumTests int `xml:"tests,attr"` + Errors int `xml:"errors,attr"` + Failures int `xml:"failures,attr"` + Props struct { + XMLName xml.Name `xml:"properties"` + Props []junitProp + } + Tests []junitTestcase +} + +func (j *junitTestsuite) Init(r *Result) error { + j.Name = r.Name + j.Errors = 0 + j.Failures = 0 + j.NumTests = 0 + j.Props.Props = make([]junitProp, 1) + j.Props.Props[0].Name = "manifest" + j.Props.Props[0].Value = r.Report.Hash.String() + + if r.Err != nil { + j.Tests = []junitTestcase{ + {Error: r.Err.Error()}, + } + j.NumTests++ + j.Errors++ + return nil + } + + j.Tests = make([]junitTestcase, len(r.Report.Packages)) + j.NumTests = len(r.Report.Packages) + i := 0 + var b strings.Builder + for pkgID, pkg := range r.Report.Packages { + tc := &j.Tests[i] + tc.Name = fmt.Sprintf("%s %s", pkg.Name, pkg.Version) + vs, ok := r.Report.PackageVulnerabilities[pkgID] + if ok { + j.Failures++ + b.Reset() + b.WriteString("Found the following vulnerabilities:") + for _, vID := range vs { + v := r.Report.Vulnerabilities[vID] + b.WriteByte('\n') + b.WriteByte('\t') + b.WriteString(v.Name) + } + tc.Failure = b.String() + } + i++ + } + return nil +} + +type junitProp struct { + XMLName xml.Name `xml:"property"` + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +type junitTestcase struct { + XMLName xml.Name `xml:"testcase"` + Name string `xml:"name,attr"` + Error string `xml:"error,omitempty"` + Failure string `xml:"failure,omitempty"` +} diff --git a/go.mod b/go.mod index 9eb0a110eb..8357cb047b 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/prometheus/procfs v0.0.8 // indirect github.com/quay/claircore v0.0.18 github.com/rs/zerolog v1.16.0 + github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 github.com/urfave/cli/v2 v2.2.0 go.opentelemetry.io/otel v0.2.1 go.opentelemetry.io/otel/exporter/metric/prometheus v0.2.2-0.20200111012159-d85178b63b15 diff --git a/go.sum b/go.sum index df497f2b08..e9f230b81b 100644 --- a/go.sum +++ b/go.sum @@ -470,6 +470,8 @@ github.com/tadasv/go-dpkg v0.0.0-20160704224136-c2cf9188b763/go.mod h1:F1AdFgT4E github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e h1:RumXZ56IrCj4CL+g1b9OL/oH0QnsF976bC8xQFYUD5Q= github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= +github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ultraware/funlen v0.0.2 h1:Av96YVBwwNSe4MLR7iI/BIa3VyI7/djnto/pK3Uxbdo=