vulnerability_matcher.go

package grype

import (
	"fmt"
	"slices"
	"strings"

	"github.com/wagoodman/go-partybus"
	"github.com/wagoodman/go-progress"

	grypeDb "github.com/anchore/grype/grype/db/v5"
	"github.com/anchore/grype/grype/distro"
	"github.com/anchore/grype/grype/event"
	"github.com/anchore/grype/grype/event/monitor"
	"github.com/anchore/grype/grype/grypeerr"
	"github.com/anchore/grype/grype/match"
	"github.com/anchore/grype/grype/matcher"
	"github.com/anchore/grype/grype/matcher/stock"
	"github.com/anchore/grype/grype/pkg"
	"github.com/anchore/grype/grype/store"
	"github.com/anchore/grype/grype/vex"
	"github.com/anchore/grype/grype/vulnerability"
	"github.com/anchore/grype/internal/bus"
	"github.com/anchore/grype/internal/log"
	"github.com/anchore/syft/syft/linux"
	syftPkg "github.com/anchore/syft/syft/pkg"
)

const (
	branch = "├──"
	leaf   = "└──"
)

type VulnerabilityMatcher struct {
	Store          store.Store
	Matchers       []matcher.Matcher
	IgnoreRules    []match.IgnoreRule
	FailSeverity   *vulnerability.Severity
	NormalizeByCVE bool
	VexProcessor   *vex.Processor
}

func DefaultVulnerabilityMatcher(store store.Store) *VulnerabilityMatcher {
	return &VulnerabilityMatcher{
		Store:    store,
		Matchers: matcher.NewDefaultMatchers(matcher.Config{}),
	}
}

func (m *VulnerabilityMatcher) FailAtOrAboveSeverity(severity *vulnerability.Severity) *VulnerabilityMatcher {
	m.FailSeverity = severity
	return m
}

func (m *VulnerabilityMatcher) WithMatchers(matchers []matcher.Matcher) *VulnerabilityMatcher {
	m.Matchers = matchers
	return m
}

func (m *VulnerabilityMatcher) WithIgnoreRules(ignoreRules []match.IgnoreRule) *VulnerabilityMatcher {
	m.IgnoreRules = ignoreRules
	return m
}

func (m *VulnerabilityMatcher) FindMatches(pkgs []pkg.Package, context pkg.Context) (remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, err error) {
	progressMonitor := trackMatcher(len(pkgs))

	defer func() {
		progressMonitor.SetCompleted()
		if err != nil {
			progressMonitor.MatchesDiscovered.SetError(err)
		}
	}()

	remainingMatches, ignoredMatches, err = m.findDBMatches(pkgs, context, progressMonitor)
	if err != nil {
		return remainingMatches, ignoredMatches, err
	}

	remainingMatches, ignoredMatches, err = m.findVEXMatches(context, remainingMatches, ignoredMatches, progressMonitor)
	if err != nil {
		err = fmt.Errorf("unable to find matches against VEX sources: %w", err)
		return remainingMatches, ignoredMatches, err
	}

	if m.FailSeverity != nil && HasSeverityAtOrAbove(m.Store, *m.FailSeverity, *remainingMatches) {
		err = grypeerr.ErrAboveSeverityThreshold
		return remainingMatches, ignoredMatches, err
	}

	logListSummary(progressMonitor)

	logIgnoredMatches(ignoredMatches)

	return remainingMatches, ignoredMatches, nil
}

func (m *VulnerabilityMatcher) findDBMatches(pkgs []pkg.Package, context pkg.Context, progressMonitor *monitorWriter) (*match.Matches, []match.IgnoredMatch, error) {
	var ignoredMatches []match.IgnoredMatch

	log.Trace("finding matches against DB")
	matches, err := m.searchDBForMatches(context.Distro, pkgs, progressMonitor)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to find matches in DB: %w", err)
	}

	matches, ignoredMatches = m.applyIgnoreRules(matches)

	if m.NormalizeByCVE {
		normalizedMatches := match.NewMatches()
		for originalMatch := range matches.Enumerate() {
			normalizedMatches.Add(m.normalizeByCVE(originalMatch))
		}

		// we apply the ignore rules again in case any of the transformations done during normalization
		// regresses the results (relative to the already applied ignore rules). Why do we additionally apply
		// the ignore rules before normalizing? In case the user has a rule that ignores a non-normalized
		// vulnerability ID, we wantMatches to ensure that the rule is honored.
		matches, ignoredMatches = m.applyIgnoreRules(normalizedMatches)
	}

	return &matches, ignoredMatches, err
}

func (m *VulnerabilityMatcher) searchDBForMatches(
	release *linux.Release,
	packages []pkg.Package,
	progressMonitor *monitorWriter,
) (match.Matches, error) {
	var err error
	res := match.NewMatches()
	matcherIndex, defaultMatcher := newMatcherIndex(m.Matchers)

	var d *distro.Distro
	if release != nil {
		d, err = distro.NewFromRelease(*release)
		if err != nil {
			log.Warnf("unable to determine linux distribution: %+v", err)
		}
		if d != nil && d.Disabled() {
			log.Warnf("unsupported linux distribution: %s", d.Name())
			return match.NewMatches(), nil
		}
	}

	distroFalsePositivesByLocationPath := make(map[string][]string)
	if d != nil && (d.Type == distro.Wolfi || d.Type == distro.Chainguard || d.Type == distro.Alpine) {
		distroFalsePositivesByLocationPath, err = indexFalsePositivesByLocation(d, packages, m.Store)
		if err != nil {
			return match.Matches{}, err
		}
	}

	if defaultMatcher == nil {
		defaultMatcher = stock.NewStockMatcher(stock.MatcherConfig{UseCPEs: true})
	}
	for _, p := range packages {
		progressMonitor.PackagesProcessed.Increment()
		log.WithFields("package", displayPackage(p)).Trace("searching for vulnerability matches")

		matchAgainst, ok := matcherIndex[p.Type]
		if !ok {
			matchAgainst = []matcher.Matcher{defaultMatcher}
		}
		for _, theMatcher := range matchAgainst {
			matches, err := theMatcher.Match(m.Store, d, p)
			if err != nil {
				log.WithFields("error", err, "package", displayPackage(p)).Warn("matcher failed")
				continue
			}

			matches = filterMatchesUsingDistroFalsePositives(matches, distroFalsePositivesByLocationPath)

			// Filter out matches based on records in the database exclusion table and hard-coded rules
			filtered, dropped := match.ApplyExplicitIgnoreRules(m.Store, match.NewMatches(matches...))

			additionalMatches := filtered.Sorted()
			logPackageMatches(p, additionalMatches)
			logExplicitDroppedPackageMatches(p, dropped)
			res.Add(additionalMatches...)

			progressMonitor.MatchesDiscovered.Add(int64(len(additionalMatches)))

			// note: there is a difference between "ignore" and "dropped" matches.
			// ignored: matches that are filtered out due to user-provided ignore rules
			// dropped: matches that are filtered out due to hard-coded rules
			updateVulnerabilityList(progressMonitor, additionalMatches, nil, dropped, m.Store)
		}
	}

	return res, nil
}

func indexFalsePositivesByLocation(
	d *distro.Distro,
	packages []pkg.Package,
	s store.Store,
) (map[string][]string, error) {
	distroFalsePositivesByLocationPath := make(map[string][]string)

	for _, p := range packages {
		falsePositivesByLocation, err := getDistroFalsePositivesByLocation(s, d, p)
		if err != nil {
			return nil, err
		}
		for l, vulnIDs := range falsePositivesByLocation {
			distroFalsePositivesByLocationPath[l] = append(distroFalsePositivesByLocationPath[l], vulnIDs...)
		}
	}

	return distroFalsePositivesByLocationPath, nil
}

func getDistroFalsePositivesByLocation(s store.Store, d *distro.Distro, p pkg.Package) (map[string][]string, error) {
	result := make(map[string][]string)

	if data, ok := p.Metadata.(pkg.ApkMetadata); ok {
		entries, err := s.GetByDistro(d, p)
		if err != nil {
			return nil, err
		}
		for _, entry := range entries {
			if entry.Constraint.String() == "< 0 (apk)" {
				for _, f := range data.Files {
					result[f.Path] = append(result[f.Path], entry.ID)
				}
			}
		}

		for _, upstreamPkg := range pkg.UpstreamPackages(p) {
			entriesForUpstream, err := s.GetByDistro(d, upstreamPkg)
			if err != nil {
				return nil, err
			}
			for _, entry := range entriesForUpstream {
				if entry.Constraint.String() == "< 0 (apk)" {
					for _, f := range data.Files {
						result[f.Path] = append(result[f.Path], entry.ID)
					}
				}
			}
		}
	}

	return result, nil
}

func filterMatchesUsingDistroFalsePositives(ms []match.Match, falsePositivesByLocation map[string][]string) []match.Match {
	var result []match.Match
	for _, m := range ms {
		isFalsePositive := false

		for _, l := range m.Package.Locations.ToSlice() {
			if fpVulnIDs, ok := falsePositivesByLocation[l.RealPath]; ok {
				if slices.Contains(fpVulnIDs, m.Vulnerability.ID) {
					isFalsePositive = true
					break
				}

				for _, relatedVulnerability := range m.Vulnerability.RelatedVulnerabilities {
					if slices.Contains(fpVulnIDs, relatedVulnerability.ID) {
						isFalsePositive = true
						break
					}
				}
			}
		}

		if !isFalsePositive {
			result = append(result, m)
		} else {
			log.WithFields("vuln", m.Vulnerability.ID, "package", displayPackage(m.Package)).Trace("dropping false positive using distro security data")
		}
	}

	return result
}

func (m *VulnerabilityMatcher) findVEXMatches(context pkg.Context, remainingMatches *match.Matches, ignoredMatches []match.IgnoredMatch, progressMonitor *monitorWriter) (*match.Matches, []match.IgnoredMatch, error) {
	if m.VexProcessor == nil {
		log.Trace("no VEX documents provided, skipping VEX matching")
		return remainingMatches, ignoredMatches, nil
	}

	log.Trace("finding matches against available VEX documents")
	matchesAfterVex, ignoredMatchesAfterVex, err := m.VexProcessor.ApplyVEX(&context, remainingMatches, ignoredMatches)
	if err != nil {
		return nil, nil, fmt.Errorf("unable to find matches against VEX documents: %w", err)
	}

	diffMatches := matchesAfterVex.Diff(*remainingMatches)
	// note: this assumes that the diff can only be additive
	diffIgnoredMatches := ignoredMatchesDiff(ignoredMatchesAfterVex, ignoredMatches)

	updateVulnerabilityList(progressMonitor, diffMatches.Sorted(), diffIgnoredMatches, nil, m.Store)

	return matchesAfterVex, ignoredMatchesAfterVex, nil
}

func (m *VulnerabilityMatcher) applyIgnoreRules(matches match.Matches) (match.Matches, []match.IgnoredMatch) {
	var ignoredMatches []match.IgnoredMatch
	if len(m.IgnoreRules) == 0 {
		return matches, ignoredMatches
	}

	matches, ignoredMatches = match.ApplyIgnoreRules(matches, m.IgnoreRules)

	if count := len(ignoredMatches); count > 0 {
		log.Infof("ignoring %d matches due to user-provided ignore rules", count)
	}
	return matches, ignoredMatches
}

func (m *VulnerabilityMatcher) normalizeByCVE(match match.Match) match.Match {
	if isCVE(match.Vulnerability.ID) {
		return match
	}

	var effectiveCVERecordRefs []vulnerability.Reference
	for _, ref := range match.Vulnerability.RelatedVulnerabilities {
		if isCVE(ref.ID) {
			effectiveCVERecordRefs = append(effectiveCVERecordRefs, ref)
			break
		}
	}

	switch len(effectiveCVERecordRefs) {
	case 0:
		log.WithFields(
			"vuln", match.Vulnerability.ID,
			"package", displayPackage(match.Package),
		).Trace("unable to find CVE record for vulnerability, skipping normalization")
		return match
	case 1:
		break
	default:
		log.WithFields(
			"refs", fmt.Sprintf("%+v", effectiveCVERecordRefs),
			"vuln", match.Vulnerability.ID,
			"package", displayPackage(match.Package),
		).Trace("found multiple CVE records for vulnerability, skipping normalization")
		return match
	}

	ref := effectiveCVERecordRefs[0]

	upstreamMetadata, err := m.Store.GetMetadata(ref.ID, ref.Namespace)
	if err != nil {
		log.WithFields("id", ref.ID, "namespace", ref.Namespace, "error", err).Warn("unable to fetch effective CVE metadata")
		return match
	}

	if upstreamMetadata == nil {
		return match
	}

	originalRef := vulnerability.Reference{
		ID:        match.Vulnerability.ID,
		Namespace: match.Vulnerability.Namespace,
	}

	match.Vulnerability.ID = upstreamMetadata.ID
	match.Vulnerability.Namespace = upstreamMetadata.Namespace
	match.Vulnerability.RelatedVulnerabilities = []vulnerability.Reference{originalRef}

	return match
}

func displayPackage(p pkg.Package) string {
	if p.PURL != "" {
		return p.PURL
	}
	return fmt.Sprintf("%s@%s (%s)", p.Name, p.Version, p.Type)
}

func ignoredMatchesDiff(subject []match.IgnoredMatch, other []match.IgnoredMatch) []match.IgnoredMatch {
	// TODO(alex): the downside with this implementation is that it does not account for the same ignored match being
	// ignored for different reasons (the appliedIgnoreRules field).

	otherMap := make(map[match.Fingerprint]struct{})
	for _, a := range other {
		otherMap[a.Match.Fingerprint()] = struct{}{}
	}

	var diff []match.IgnoredMatch
	for _, b := range subject {
		if _, ok := otherMap[b.Match.Fingerprint()]; !ok {
			diff = append(diff, b)
		}
	}

	return diff
}

func newMatcherIndex(matchers []matcher.Matcher) (map[syftPkg.Type][]matcher.Matcher, matcher.Matcher) {
	matcherIndex := make(map[syftPkg.Type][]matcher.Matcher)
	var defaultMatcher matcher.Matcher
	for _, m := range matchers {
		if m.Type() == match.StockMatcher {
			defaultMatcher = m
			continue
		}
		for _, t := range m.PackageTypes() {
			if _, ok := matcherIndex[t]; !ok {
				matcherIndex[t] = make([]matcher.Matcher, 0)
			}

			matcherIndex[t] = append(matcherIndex[t], m)
			log.Debugf("adding matcher: %+v", t)
		}
	}

	return matcherIndex, defaultMatcher
}

func isCVE(id string) bool {
	return strings.HasPrefix(strings.ToLower(id), "cve-")
}

func HasSeverityAtOrAbove(store vulnerability.MetadataProvider, severity vulnerability.Severity, matches match.Matches) bool {
	if severity == vulnerability.UnknownSeverity {
		return false
	}
	for m := range matches.Enumerate() {
		metadata, err := store.GetMetadata(m.Vulnerability.ID, m.Vulnerability.Namespace)
		if err != nil {
			continue
		}

		if vulnerability.ParseSeverity(metadata.Severity) >= severity {
			return true
		}
	}
	return false
}

func logListSummary(vl *monitorWriter) {
	log.Infof("found %d vulnerability matches across %d packages", vl.MatchesDiscovered.Current(), vl.PackagesProcessed.Current())
	log.Debugf("  ├── fixed: %d", vl.Fixed.Current())
	log.Debugf("  ├── ignored: %d (due to user-provided rule)", vl.Ignored.Current())
	log.Debugf("  ├── dropped: %d (due to hard-coded correction)", vl.Dropped.Current())
	log.Debugf("  └── matched: %d", vl.MatchesDiscovered.Current())

	var unknownCount int64
	if count, ok := vl.BySeverity[vulnerability.UnknownSeverity]; ok {
		unknownCount = count.Current()
	}
	log.Debugf("      ├── %s: %d", vulnerability.UnknownSeverity.String(), unknownCount)

	allSeverities := vulnerability.AllSeverities()
	for idx, sev := range allSeverities {
		arm := selectArm(idx, len(allSeverities))
		log.Debugf("      %s %s: %d", arm, sev.String(), vl.BySeverity[sev].Current())
	}
}

func updateVulnerabilityList(mon *monitorWriter, matches []match.Match, ignores []match.IgnoredMatch, dropped []match.IgnoredMatch, metadataProvider vulnerability.MetadataProvider) {
	for _, m := range matches {
		metadata, err := metadataProvider.GetMetadata(m.Vulnerability.ID, m.Vulnerability.Namespace)
		if err != nil || metadata == nil {
			mon.BySeverity[vulnerability.UnknownSeverity].Increment()
			continue
		}

		sevManualProgress, ok := mon.BySeverity[vulnerability.ParseSeverity(metadata.Severity)]
		if !ok {
			mon.BySeverity[vulnerability.UnknownSeverity].Increment()
			continue
		}
		sevManualProgress.Increment()

		if m.Vulnerability.Fix.State == grypeDb.FixedState {
			mon.Fixed.Increment()
		}
	}

	mon.Ignored.Add(int64(len(ignores)))
	mon.Dropped.Add(int64(len(dropped)))
}

func logPackageMatches(p pkg.Package, matches []match.Match) {
	if len(matches) == 0 {
		return
	}

	log.WithFields("package", displayPackage(p)).Debugf("found %d vulnerabilities", len(matches))
	for idx, m := range matches {
		arm := selectArm(idx, len(matches))
		log.WithFields("vuln", m.Vulnerability.ID, "namespace", m.Vulnerability.Namespace).Debugf("  %s", arm)
	}
}

func selectArm(idx, total int) string {
	if idx == total-1 {
		return leaf
	}
	return branch
}

func logExplicitDroppedPackageMatches(p pkg.Package, ignored []match.IgnoredMatch) {
	if len(ignored) == 0 {
		return
	}

	log.WithFields("package", displayPackage(p)).Debugf("dropped %d vulnerability matches due to hard-coded correction", len(ignored))
	for idx, i := range ignored {
		arm := selectArm(idx, len(ignored))

		log.WithFields("vuln", i.Match.Vulnerability.ID, "rules", len(i.AppliedIgnoreRules)).Debugf("  %s", arm)
	}
}

func logIgnoredMatches(ignored []match.IgnoredMatch) {
	if len(ignored) == 0 {
		return
	}

	log.Infof("ignored %d vulnerability matches", len(ignored))
	for idx, i := range ignored {
		arm := selectArm(idx, len(ignored))

		log.WithFields("vuln", i.Match.Vulnerability.ID, "rules", len(i.AppliedIgnoreRules), "package", displayPackage(i.Package)).Debugf("  %s", arm)
	}
}

type monitorWriter struct {
	PackagesProcessed *progress.Manual
	MatchesDiscovered *progress.Manual
	Fixed             *progress.Manual
	Ignored           *progress.Manual
	Dropped           *progress.Manual
	BySeverity        map[vulnerability.Severity]*progress.Manual
}

func newMonitor(pkgCount int) (monitorWriter, monitor.Matching) {
	manualBySev := make(map[vulnerability.Severity]*progress.Manual)
	for _, severity := range vulnerability.AllSeverities() {
		manualBySev[severity] = progress.NewManual(-1)
	}
	manualBySev[vulnerability.UnknownSeverity] = progress.NewManual(-1)

	m := monitorWriter{
		PackagesProcessed: progress.NewManual(int64(pkgCount)),
		MatchesDiscovered: progress.NewManual(-1),
		Fixed:             progress.NewManual(-1),
		Ignored:           progress.NewManual(-1),
		Dropped:           progress.NewManual(-1),
		BySeverity:        manualBySev,
	}

	monitorableBySev := make(map[vulnerability.Severity]progress.Monitorable)
	for sev, manual := range manualBySev {
		monitorableBySev[sev] = manual
	}

	return m, monitor.Matching{
		PackagesProcessed: m.PackagesProcessed,
		MatchesDiscovered: m.MatchesDiscovered,
		Fixed:             m.Fixed,
		Ignored:           m.Ignored,
		Dropped:           m.Dropped,
		BySeverity:        monitorableBySev,
	}
}

func (m *monitorWriter) SetCompleted() {
	m.PackagesProcessed.SetCompleted()
	m.MatchesDiscovered.SetCompleted()
	m.Fixed.SetCompleted()
	m.Ignored.SetCompleted()
	m.Dropped.SetCompleted()
	for _, v := range m.BySeverity {
		v.SetCompleted()
	}
}

func trackMatcher(pkgCount int) *monitorWriter {
	writer, reader := newMonitor(pkgCount)

	bus.Publish(partybus.Event{
		Type:  event.VulnerabilityScanningStarted,
		Value: reader,
	})

	return &writer
}