Louis/dedupe vulns remodel (#93)

debian/matcher_integration_test.go

@@ -9,8 +9,8 @@ import (
 	"time"
 
 	"github.com/quay/claircore"
+	"github.com/quay/claircore/internal/matcher"
 	"github.com/quay/claircore/internal/updater"
-	"github.com/quay/claircore/internal/vulnscanner"
 	vulnstore "github.com/quay/claircore/internal/vulnstore/postgres"
 	"github.com/quay/claircore/libvuln/driver"
 	distlock "github.com/quay/claircore/pkg/distlock/postgres"
@@ -29,12 +29,9 @@ func Test_Matcher_Integration(t *testing.T) {
 	ctx, _ = log.TestLogger(ctx, t)
 	db, store, _, teardown := vulnstore.TestStore(ctx, t)
 	defer teardown()
-
 	m := &Matcher{}
-
 	// seed the test vulnstore with CVE data
 	deb := NewUpdater(Buster)
-
 	up := updater.New(&updater.Opts{
 		Name:    "test-debian-buster",
 		Updater: deb,
@@ -47,25 +44,21 @@ func Test_Matcher_Integration(t *testing.T) {
 	ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
 	defer cancel()
 	up.Update(ctx)
-
 	path := filepath.Join("testdata", "indexreport-buster-jackson-databind.json")
 	f, err := os.Open(path)
 	if err != nil {
 		t.Fatalf("%v", err)
 	}
 
-	var sr claircore.IndexReport
-	err = json.NewDecoder(f).Decode(&sr)
+	var ir claircore.IndexReport
+	err = json.NewDecoder(f).Decode(&ir)
 	if err != nil {
 		t.Fatalf("failed to decode IndexReport: %v", err)
 	}
-
-	vs := vulnscanner.New(store, []driver.Matcher{m})
-	vr, err := vs.Scan(ctx, &sr)
+	vr, err := matcher.Match(context.Background(), &ir, []driver.Matcher{m}, store)
 	if err != nil {
-		t.Error(err)
+		t.Fatalf("expected nil error but got %v", err)
 	}
-
 	_, err = json.Marshal(&vr)
 	if err != nil {
 		t.Fatalf("failed to marshal VR: %v", err)

distribution.go

@@ -1,8 +1,9 @@
 package claircore
 
 // Distribution is the accompanying system context of a package. this
-// information aides in CVE detection. scanners should identify this information before
-// starting their scan and tag each found package with as much as this discovered info as possible.
+// information aides in CVE detection.
+//
+// Distribution is modeled after the os-release file found in all linux distributions.
 type Distribution struct {
 	// unique ID of this distribution. this will be created as discovered by the library
 	// and used for persistence and hash map indexes.

docs/matching_vulns.md

@@ -132,10 +132,10 @@ The default scanner works as follows:
 
 ## The vulnerability matching process
 
-Matching vulnerabilities is facilitated by methods in the `claircore.internal.vulnstore`, `claircore.internal.matcher`, and `claircore.internal.vulnscanner` packages. They process looks like this:  
+Matching vulnerabilities is facilitated by methods in the `claircore.internal.vulnstore` and `claircore.internal.matcher` packages. They process looks like this:  
 
 1. libvuln is instantiated and configured with a set of `claircore.internal.matcher` implementations. 
 2. lubvuln gets a request to find vulnerabilities for a manifest. first it reaches out to libindex to retrieve the IndexReport
-3. with the IndexReport retrieved a `claircore.internal.vulnscanner.VulnScanner` is created.
+3. with the IndexReport retrieved a call to `claircore.internal.matcher.Match` is made.
 4. the VulnScanner launches all configured Matcher(s) by way of a MatchController. The MatchController drives the Matcher(s) calling the appropriate functions and handling results and errors
 5. the VulnScanner dedupes and merges all vulnerabilities discovered by MatchControllers and returns a VulernabilityReport

environment.go

@@ -0,0 +1,18 @@
+package claircore
+
+// Environment describes the surrounding environment a package was
+// discovered in.
+//
+// Environment must be accompanied by a parent structure which maps
+// IDs to data models in order to have meaning. In our case this is
+// IndexReport or VulnerabilityReport.
+type Environment struct {
+	// the package database the associated package was discovered in
+	PackageDB string `json:"package_db"`
+	// the layer in which the associated package was introduced
+	IntroducedIn string `json:"introduced_in"`
+	// the ID of the distribution the package was discovered on
+	DistributionID int `json:"distribution_id"`
+	// the ID of the repository where this package was downloaded from (currently not used)
+	RepositoryID int `json:"repository_id"`
+}

indexreport.go

@@ -2,67 +2,50 @@ package claircore
 
 // IndexRecord is an entry in the IndexReport.
 //
-// A IndexRecord identifies a discovered package along with its
-// Distribution and Repository information if present.
+// IndexRecords provide full access to contextual package
+// structures such as Distribution and Repository.
+//
+// A list of these can be thought of as an "unpacked" IndexReport
 type IndexRecord struct {
 	Package      *Package
 	Distribution *Distribution
 	Repository   *Repository
 }
 
-// IndexReport provides a package database for a container image.
+// IndexReport provides a database for discovered artifacts in an image.
 //
-// A IndexReport is used to inventory a discrete package information found
-// within in each layer of a container image.
+// IndexReports make heavy usage of lookup maps to associate information
+// without repetition.
 type IndexReport struct {
-	// the manifest hash this scan result is assocaited with
+	// the manifest hash this IndexReport is describing
 	Hash string `json:"manifest_hash"`
-	// the current state of the scan.
+	// the current state of the index operation
 	State string `json:"state"`
-	// packages found after applying all layers
+	// all discovered packages in this manifest key'd by package id
 	Packages map[int]*Package `json:"packages"`
-	// distributions found after applying all layers
+	// all discovered distributions in this manifest key'd by distribution id
 	Distributions map[int]*Distribution `json:"distributions"`
-	// repositories found after applying all layers
+	// all discovered repositories in this manifest key'd by repository id
 	Repositories map[int]*Repository `json:"repository"`
-	// PackagesByDistribution maps a package id to it's associated distribution id
-	DistributionByPackage map[int]int `json:"distribution_by_package"`
-	// PackagesByRepositories maps a package id to it's associated repository id
-	RepositoryByPackage map[int]int `json:"repository_by_package"`
-	// layer hash that introduced the given package id
-	PackageIntroduced map[int]string `json:"package_introduced"`
-	// whether the scan was successful
+	// a list of environment details a package was discovered in key'd by package id
+	Environments map[int][]*Environment `json:"environments"`
+	// whether the index operation finished successfully
 	Success bool `json:"success"`
-	// the first fatal error that occured during a scan process
+	// an error string in the case the index did not succeed
 	Err string `json:"err"`
 }
 
 // IndexRecords returns a list of IndexRecords derived from the IndexReport
-//
-// If a field in the IndexRecord is not found in the IndexReport the empty value
-// is returned to provide nil safey.
 func (report *IndexReport) IndexRecords() []*IndexRecord {
 	out := []*IndexRecord{}
 	for _, pkg := range report.Packages {
-		record := &IndexRecord{}
-		record.Package = pkg
-
-		if record.Package.Source == nil {
-			record.Package.Source = &Package{}
-		}
-
-		if id, ok := report.DistributionByPackage[pkg.ID]; ok {
-			record.Distribution = report.Distributions[id]
-		} else {
-			record.Distribution = &Distribution{}
-		}
-
-		if id, ok := report.RepositoryByPackage[pkg.ID]; ok {
-			record.Repository = report.Repositories[id]
-		} else {
-			record.Repository = &Repository{}
+		for _, env := range report.Environments[pkg.ID] {
+			record := &IndexRecord{}
+			record.Package = pkg
+			record.Distribution = report.Distributions[env.DistributionID]
+			record.Repository = report.Repositories[env.RepositoryID]
+			out = append(out, record)
 		}
-		out = append(out, record)
 	}
 	return out
 }

internal/indexer/controller/coalesce.go

@@ -53,30 +53,21 @@ func coalesce(ctx context.Context, s *Controller) (State, error) {
 // source is the IndexReport that the indexer is working on.
 // merge is an array IndexReports returned from coalescers
 func MergeSR(source *claircore.IndexReport, merge []*claircore.IndexReport) *claircore.IndexReport {
-	for _, sr := range merge {
-		for k, v := range sr.Packages {
+	for _, ir := range merge {
+		for k, v := range ir.Environments {
+			source.Environments[k] = append(source.Environments[k], v...)
+		}
+		for k, v := range ir.Packages {
 			source.Packages[k] = v
 		}
 
-		for k, v := range sr.Distributions {
+		for k, v := range ir.Distributions {
 			source.Distributions[k] = v
 		}
 
-		for k, v := range sr.Repositories {
+		for k, v := range ir.Repositories {
 			source.Repositories[k] = v
 		}
-
-		for k, v := range sr.DistributionByPackage {
-			source.DistributionByPackage[k] = v
-		}
-
-		for k, v := range sr.RepositoryByPackage {
-			source.RepositoryByPackage[k] = v
-		}
-
-		for k, v := range sr.PackageIntroduced {
-			source.PackageIntroduced[k] = v
-		}
 	}
 	return source
 }

internal/indexer/controller/controller.go

@@ -84,12 +84,10 @@ type Controller struct {
 func New(opts *indexer.Opts) *Controller {
 	// fully init any maps and arrays
 	scanRes := &claircore.IndexReport{
-		PackageIntroduced:     map[int]string{},
-		Packages:              map[int]*claircore.Package{},
-		Distributions:         map[int]*claircore.Distribution{},
-		Repositories:          map[int]*claircore.Repository{},
-		DistributionByPackage: map[int]int{},
-		RepositoryByPackage:   map[int]int{},
+		Packages:      map[int]*claircore.Package{},
+		Environments:  map[int][]*claircore.Environment{},
+		Distributions: map[int]*claircore.Distribution{},
+		Repositories:  map[int]*claircore.Repository{},
 	}
 
 	s := &Controller{