rhel: add csaf/vex updater

Replace the Red Hat OVALv2 update source with the Red Hat CSAF/VEX data. Signed-off-by: crozzy <joseph.crosland@gmail.com>
quay · Jan 23, 2024 · 89006e1 · 89006e1
1 parent e0d5be9
commit 89006e1
Show file tree

Hide file tree

Showing 17 changed files with 300,615 additions and 0 deletions.
diff --git a/go.mod b/go.mod
@@ -15,6 +15,7 @@ require (
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f
 	github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d
 	github.com/knqyf263/go-rpm-version v0.0.0-20170716094938-74609b86c936
+	github.com/package-url/packageurl-go v0.1.2
 	github.com/prometheus/client_golang v1.18.0
 	github.com/quay/claircore/toolkit v1.1.1
 	github.com/quay/claircore/updater/driver v1.0.0

diff --git a/go.sum b/go.sum
@@ -136,6 +136,8 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
 github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
+github.com/package-url/packageurl-go v0.1.2 h1:0H2DQt6DHd/NeRlVwW4EZ4oEI6Bn40XlNPRqegcxuo4=
+github.com/package-url/packageurl-go v0.1.2/go.mod h1:uQd4a7Rh3ZsVg5j0lNyAfyxIeGde9yrlhjF78GzeW0c=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

diff --git a/rhel/vex/fetcher.go b/rhel/vex/fetcher.go
@@ -0,0 +1,279 @@
+package vex
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore/libvuln/driver"
+	"github.com/quay/claircore/pkg/tmp"
+)
+
+// A Changes is a series of Entries indicating where to find resources in the
+// changes.csv file.
+type Changes []Entry
+
+// Entry is an entry in a changes.csv file.
+type Entry struct {
+	// This path should be parsed in the context of the manifest's URL.
+	Path        string
+	updatedTime time.Time
+}
+
+func (u *VEXUpdater) Fetch(ctx context.Context, hint driver.Fingerprint) (io.ReadCloser, driver.Fingerprint, error) {
+	ctx = zlog.ContextWithValues(ctx, "component", "rhel/VEXUpdater.Fetch")
+	fp, err := ParseFingerprint(string(hint))
+	if err != nil {
+		return nil, hint, err
+	}
+
+	f, err := tmp.NewFile("", "rhel-vex.")
+	if err != nil {
+		return nil, hint, err
+	}
+
+	var success bool
+	defer func() {
+		if success {
+			if _, err := f.Seek(0, io.SeekStart); err != nil {
+				zlog.Warn(ctx).
+					Err(err).
+					Msg("unable to seek file back to start")
+			}
+		} else {
+			if err := f.Close(); err != nil {
+				zlog.Warn(ctx).Err(err).Msg("unable to close spool")
+			}
+		}
+	}()
+
+	if fp.changesEtag == "" { // Used to inform whether this is the first run.
+		// We need to go after the full corpus of vulnerabilities
+		// First we target the archive_latest.txt file
+		latestURI, err := u.url.Parse(latestFile)
+		if err != nil {
+			return nil, hint, err
+		}
+		latestReq, err := http.NewRequestWithContext(ctx, http.MethodGet, latestURI.String(), nil)
+		if err != nil {
+			return nil, hint, err
+		}
+		latestRes, err := u.client.Do(latestReq)
+		if latestRes != nil {
+			defer latestRes.Body.Close()
+		}
+		if err != nil {
+			return nil, hint, err
+		}
+		if latestRes.StatusCode != http.StatusOK {
+			return nil, hint, fmt.Errorf("unexpected response from archive_latest.txt: %s", latestRes.Status)
+		}
+
+		body, err := io.ReadAll(latestRes.Body) // Fine to use as expecting small number of bytes.
+		if err != nil {
+			return nil, hint, err
+		}
+
+		compressedFilename := string(body)
+		zlog.Debug(ctx).
+			Str("filename", compressedFilename).
+			Msg("requesting latest compressed file")
+
+		// Decided against parsing the filename to allow the implementation
+		// to change under the hood w/o Clair caring.
+		uri, err := u.url.Parse(compressedFilename)
+		if err != nil {
+			return nil, hint, err
+		}
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri.String(), nil)
+		if err != nil {
+			return nil, hint, err
+		}
+
+		u.client.Timeout = time.Minute * 1 // This is a large request.
+		res, err := u.client.Do(req)
+		if res != nil {
+			defer res.Body.Close()
+		}
+		if err != nil {
+			return nil, hint, err
+		}
+		if res.StatusCode != http.StatusOK {
+			return nil, hint, fmt.Errorf("unexpected response from latest compressed file: %s", res.Status)
+		}
+
+		lm := res.Header.Get("last-modified")
+		fp.requestTime, err = time.Parse(time.RFC1123, lm)
+		if err != nil {
+			return nil, hint, fmt.Errorf("could not parse last-modified header %s:, %w", lm, err)
+		}
+		z, err := zstd.NewReader(res.Body)
+		if err != nil {
+			return nil, hint, err
+		}
+		defer z.Close()
+		r := tar.NewReader(z)
+
+		var h *tar.Header
+		var buf bytes.Buffer
+		var entriesWritten int
+		for h, err = r.Next(); err == nil; h, err = r.Next() {
+			if h.Typeflag != tar.TypeReg {
+				continue
+			}
+			year, err := strconv.ParseInt(path.Dir(h.Name), 10, 64)
+			if err != nil {
+				return nil, hint, fmt.Errorf("error parsing year %w", err)
+			}
+			if year < lookBackToYear {
+				continue
+			}
+			buf.Grow(int(h.Size))
+			if _, err := buf.ReadFrom(r); err != nil {
+				return nil, hint, err
+			}
+
+			bc := &bytes.Buffer{}
+			err = json.Compact(bc, buf.Bytes())
+			if err != nil {
+				return nil, hint, fmt.Errorf("error compressing JSON %s: %w", h.Name, err)
+			}
+			bc.WriteByte('\n')
+			f.Write(bc.Bytes())
+			buf.Reset()
+			bc.Reset()
+			entriesWritten++
+		}
+		if err != io.EOF {
+			return nil, hint, fmt.Errorf("error reading tar contents: %w", err)
+		}
+
+		zlog.Debug(ctx).
+			Str("updater", u.Name()).
+			Int("entries written", entriesWritten).
+			Msg("finished writing compressed data to spool")
+
+	}
+
+	uri, err := u.url.Parse("changes.csv")
+	if err != nil {
+		return nil, hint, err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, uri.String(), nil)
+	if err != nil {
+		return nil, hint, err
+	}
+	if fp.changesEtag != "" {
+		req.Header.Add("If-None-Match", fp.changesEtag)
+	}
+	res, err := u.client.Do(req)
+	if res != nil {
+		defer res.Body.Close()
+	}
+	if err != nil {
+		return nil, hint, err
+	}
+
+	switch res.StatusCode {
+	case http.StatusOK:
+		if t := fp.changesEtag; t == "" || t != res.Header.Get("etag") {
+			break
+		}
+		fallthrough
+	case http.StatusNotModified:
+		// We could return driver.Unchanged here but we don't know for sure. Return the
+		// file that may have data read from the compressed file in it.
+		return f, hint, nil
+	default:
+		return nil, hint, fmt.Errorf("unexpected response from changes.csv: %s", res.Status)
+	}
+	fp.changesEtag = res.Header.Get("etag")
+
+	rd := csv.NewReader(res.Body)
+	rd.FieldsPerRecord = 2
+	rd.ReuseRecord = true
+	l := 0
+	rec, err := rd.Read()
+	for ; err == nil; rec, err = rd.Read() {
+		if len(rec) != 2 {
+			return nil, hint, fmt.Errorf("could not parse changes.csv file")
+		}
+		var err error
+		e := Entry{}
+		e.Path += rec[0] // This += should result in us getting a copy.
+		year, err := strconv.ParseInt(path.Dir(e.Path), 10, 64)
+		if err != nil {
+			return nil, hint, fmt.Errorf("error parsing year %w", err)
+		}
+		if year < lookBackToYear {
+			continue
+		}
+		e.updatedTime, err = time.Parse(time.RFC3339, rec[1])
+		if err != nil {
+			return nil, hint, fmt.Errorf("line %d: %w", l, err)
+		}
+		if e.updatedTime.After(fp.requestTime) {
+			en := e
+			advisoryURI, err := u.url.Parse(en.Path)
+			if err != nil {
+				return nil, hint, err
+			}
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, advisoryURI.String(), nil)
+			if err != nil {
+				return nil, hint, fmt.Errorf("error creating advisory request %w", err)
+			}
+
+			res, err := u.client.Do(req)
+			if err != nil {
+				return nil, hint, fmt.Errorf("error making advisory request %w", err)
+			}
+			if res.StatusCode != http.StatusOK {
+				var b strings.Builder
+				if _, err := io.Copy(&b, res.Body); err != nil {
+					zlog.Warn(ctx).Err(err).Msg("additional error while reading error response")
+				} else {
+					zlog.Warn(ctx).Str("response", b.String()).Msg("received error response")
+				}
+				return nil, hint, fmt.Errorf("unexpected response from advisary URL: %s %s", res.Status, req.URL)
+			}
+
+			buf := new(bytes.Buffer)
+			_, err = buf.ReadFrom(res.Body)
+			if err != nil {
+				return nil, hint, fmt.Errorf("error reading from buffer: %w", err)
+			}
+			zlog.Debug(ctx).Str("url", advisoryURI.String()).Msg("copying body to file")
+			bc := &bytes.Buffer{}
+			err = json.Compact(bc, buf.Bytes())
+			if err != nil {
+				return nil, hint, fmt.Errorf("error compressing JSON: %w", err)
+			}
+			bc.WriteByte('\n')
+			f.Write(bc.Bytes())
+			l++
+			res.Body.Close()
+		}
+	}
+
+	switch {
+	case err == io.EOF, err == nil:
+	default:
+		return nil, hint, fmt.Errorf("error parsing the csv file: %w", err)
+	}
+
+	fp.requestTime = time.Now()
+	success = true
+	return f, driver.Fingerprint(fp.String()), nil
+}