updater: port updater and its fetchers

api/handlers.go

@@ -46,8 +46,8 @@ func GETVersions(w http.ResponseWriter, r *http.Request, _ httprouter.Params, _
 }
 
 // GETHealth sums up the health of all the registered services.
-func GETHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params, _ *Env) {
-	globalHealth, statuses := health.Healthcheck()
+func GETHealth(w http.ResponseWriter, r *http.Request, _ httprouter.Params, e *Env) {
+	globalHealth, statuses := health.Healthcheck(e.Datastore)
 
 	httpStatus := http.StatusOK
 	if !globalHealth {

clair.go

@@ -25,6 +25,7 @@ import (
 	"github.com/coreos/clair/api"
 	"github.com/coreos/clair/config"
 	"github.com/coreos/clair/database/pgsql"
+	"github.com/coreos/clair/updater"
 	"github.com/coreos/clair/utils"
 	"github.com/coreos/pkg/capnslog"
 )
@@ -55,8 +56,8 @@ func Boot(config *config.Config) {
 	go api.RunHealth(config.API, &api.Env{Datastore: db}, st)
 
 	// Start updater
-	// st.Begin()
-	// go updater.Run(config.Updater, st)
+	st.Begin()
+	go updater.Run(config.Updater, db, st)
 
 	// Wait for interruption and shutdown gracefully.
 	waitForSignals(os.Interrupt)

database/pgsql/feature.go

@@ -146,7 +146,6 @@ func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.Fea
 		if featureVersion.Version.Compare(fixedInVersion) < 0 {
 			// The version of the FeatureVersion we are inserting is lower than the fixed version on this
 			// Vulnerability, thus, this FeatureVersion is affected by it.
-			// TODO(Quentin-M): Prepare.
 			_, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID,
 				featureVersion.ID, fixedInID)
 			if err != nil {

database/pgsql/pgsql.go

@@ -99,7 +99,7 @@ func createDatabase(dataSource, databaseName string) error {
 	defer db.Close()
 
 	// Create database.
-	_, err = db.Exec("CREATE DATABASE " + databaseName + ";")
+	_, err = db.Exec("CREATE DATABASE " + databaseName)
 	if err != nil {
 		return fmt.Errorf("could not create database: %v", err)
 	}
@@ -118,7 +118,7 @@ func dropDatabase(dataSource, databaseName string) error {
 	defer db.Close()
 
 	// Drop database.
-	_, err = db.Exec("DROP DATABASE " + databaseName + ";")
+	_, err = db.Exec("DROP DATABASE " + databaseName)
 	if err != nil {
 		return fmt.Errorf("could not drop database: %v", err)
 	}
@@ -167,7 +167,7 @@ func OpenForTest(name string, withTestData bool) (*pgSQLTest, error) {
 		d, _ := ioutil.ReadFile(path.Join(path.Dir(filename)) + "/testdata/data.sql")
 		_, err = db.(*pgSQL).Exec(string(d))
 		if err != nil {
-			dropDatabase(dataSource, dbName)
+			dropDatabase(dataSource+"dbname=postgres", dbName)
 			log.Error(err)
 			return nil, database.ErrCantOpen
 		}

database/pgsql/vulnerability.go

@@ -2,6 +2,7 @@ package pgsql
 
 import (
 	"database/sql"
+	"fmt"
 
 	"github.com/coreos/clair/database"
 	"github.com/coreos/clair/utils"
@@ -69,6 +70,7 @@ func (pgSQL *pgSQL) InsertVulnerabilities(vulnerabilities []database.Vulnerabili
 	for _, vulnerability := range vulnerabilities {
 		err := pgSQL.insertVulnerability(vulnerability)
 		if err != nil {
+			fmt.Printf("%#v\n", vulnerability)
 			return err
 		}
 	}
@@ -121,16 +123,18 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
 		return nil
 	}
 
-	// Insert or find the new FeatureVersions.
+	// Insert or find the new Feature.
 	// We already have the Feature IDs in updatedFixedInFeatureVersions because diffFixedIn fills them
 	// in using the existing vulnerability's FixedIn FeatureVersions. Note that even if FixedIn
 	// is type FeatureVersion, the actual stored ID in these structs are the Feature IDs.
 	//
 	// Also, we enforce the namespace of the FeatureVersion in case it was empty. There is a test
 	// above to ensure that the passed Namespace is either the same as the vulnerability or empty.
+	//
+	// TODO(Quentin-M): Batch me.
 	for i := 0; i < len(newFixedInFeatureVersions); i++ {
 		newFixedInFeatureVersions[i].Feature.Namespace.Name = vulnerability.Namespace.Name
-		newFixedInFeatureVersions[i].ID, err = pgSQL.insertFeatureVersion(newFixedInFeatureVersions[i])
+		newFixedInFeatureVersions[i].Feature.ID, err = pgSQL.insertFeature(newFixedInFeatureVersions[i].Feature)
 		if err != nil {
 			return err
 		}
@@ -229,15 +233,23 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability
 	var fixedInID int
 
 	for _, fv := range newFixedInFeatureVersions {
+		if fv.Version == types.MinVersion {
+			// We don't want to mark a Feature as fixed in MinVersion. MinVersion only makes sense when a
+			// Feature is already marked as fixed in some version, in which case we would be in the
+			// "updatedFixedInFeatureVersions" loop and removes the fixed in mark.
+			continue
+		}
+
 		// Insert Vulnerability_FixedIn_Feature.
-		err := tx.QueryRow(getQuery("i_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID,
+		err := tx.QueryRow(getQuery("i_vulnerability_fixedin_feature"), vulnerability.ID, fv.Feature.ID,
 			&fv.Version).Scan(&fixedInID)
 		if err != nil {
 			return handleError("i_vulnerability_fixedin_feature", err)
 		}
 
 		// Insert Vulnerability_Affects_FeatureVersion.
-		err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.ID, fv.Version)
+		err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.Feature.ID,
+			fv.Version)
 		if err != nil {
 			return err
 		}
@@ -246,8 +258,8 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability
 	for _, fv := range updatedFixedInFeatureVersions {
 		if fv.Version != types.MinVersion {
 			// Update Vulnerability_FixedIn_Feature.
-			err := tx.QueryRow(getQuery("u_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID,
-				&fv.Version).Scan(&fixedInID)
+			err := tx.QueryRow(getQuery("u_vulnerability_fixedin_feature"), vulnerability.ID,
+				fv.Feature.ID, &fv.Version).Scan(&fixedInID)
 			if err != nil {
 				return handleError("u_vulnerability_fixedin_feature", err)
 			}
@@ -259,23 +271,26 @@ func (pgSQL *pgSQL) updateVulnerabilityFeatureVersions(tx *sql.Tx, vulnerability
 			}
 
 			// Insert Vulnerability_Affects_FeatureVersion.
-			err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.ID, fv.Version)
+			err = linkVulnerabilityToFeatureVersions(tx, fixedInID, vulnerability.ID, fv.Feature.ID,
+				fv.Version)
 			if err != nil {
 				return err
 			}
 		} else {
 			// Updating FixedIn by saying that the fixed version is the lowest possible version, it
 			// basically means that the vulnerability doesn't affect the feature (anymore).
 			// Drop it from Vulnerability_FixedIn_Feature and Vulnerability_Affects_FeatureVersion.
-			err := tx.QueryRow(getQuery("r_vulnerability_fixedin_feature"), vulnerability.ID, fv.ID).
-				Scan(&fixedInID)
-			if err != nil {
+			err := tx.QueryRow(getQuery("r_vulnerability_fixedin_feature"), vulnerability.ID,
+				fv.Feature.ID).Scan(&fixedInID)
+			if err != nil && err != sql.ErrNoRows {
 				return handleError("r_vulnerability_fixedin_feature", err)
 			}
 
-			_, err = tx.Exec(getQuery("r_vulnerability_affects_featureversion"), fixedInID)
-			if err != nil {
-				return handleError("r_vulnerability_affects_featureversion", err)
+			if err == nil {
+				_, err = tx.Exec(getQuery("r_vulnerability_affects_featureversion"), fixedInID)
+				if err != nil {
+					return handleError("r_vulnerability_affects_featureversion", err)
+				}
 			}
 		}
 	}
@@ -304,7 +319,7 @@ func linkVulnerabilityToFeatureVersions(tx *sql.Tx, fixedInID, vulnerabilityID,
 		}
 
 		if featureVersionVersion.Compare(fixedInVersion) < 0 {
-			_, err := tx.Exec("i_vulnerability_affects_featureversion", vulnerabilityID, featureVersionID,
+			_, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID, featureVersionID,
 				fixedInID)
 			if err != nil {
 				return handleError("i_vulnerability_affects_featureversion", err)

health/health.go

@@ -19,6 +19,8 @@ package health
 import (
 	"fmt"
 	"sync"
+
+	"github.com/coreos/clair/database"
 )
 
 // Status defines a way to know the health status of a service
@@ -33,7 +35,7 @@ type Status struct {
 }
 
 // A Healthchecker function is a method returning the Status of the tested service
-type Healthchecker func() Status
+type Healthchecker func(database.Datastore) Status
 
 var (
 	healthcheckersLock sync.Mutex
@@ -59,12 +61,12 @@ func RegisterHealthchecker(name string, f Healthchecker) {
 }
 
 // Healthcheck calls every registered Healthchecker and summarize their output
-func Healthcheck() (bool, map[string]interface{}) {
+func Healthcheck(datastore database.Datastore) (bool, map[string]interface{}) {
 	globalHealth := true
 
 	statuses := make(map[string]interface{})
 	for serviceName, serviceChecker := range healthcheckers {
-		status := serviceChecker()
+		status := serviceChecker(datastore)
 
 		globalHealth = globalHealth && (!status.IsEssential || status.IsHealthy)
 		statuses[serviceName] = struct {

updater/fetchers.go

@@ -20,16 +20,15 @@ var fetchers = make(map[string]Fetcher)
 
 // Fetcher represents anything that can fetch vulnerabilities.
 type Fetcher interface {
-	FetchUpdate() (FetcherResponse, error)
+	FetchUpdate(database.Datastore) (FetcherResponse, error)
 }
 
 // FetcherResponse represents the sum of results of an update.
 type FetcherResponse struct {
 	FlagName        string
 	FlagValue       string
 	Notes           []string
-	Vulnerabilities []*database.Vulnerability
-	Packages        []*database.Package
+	Vulnerabilities []database.Vulnerability
 }
 
 // RegisterFetcher makes a Fetcher available by the provided name.

updater/fetchers/debian/debian.go

@@ -27,6 +27,7 @@ import (
 	"github.com/coreos/clair/updater"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
+	"github.com/coreos/pkg/capnslog"
 )
 
 const (
@@ -35,6 +36,8 @@ const (
 	debianUpdaterFlag = "debianUpdater"
 )
 
+var log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/debian")
+
 type jsonData map[string]map[string]jsonVuln
 
 type jsonVuln struct {
@@ -57,7 +60,7 @@ func init() {
 }
 
 // FetchUpdate fetches vulnerability updates from the Debian Security Tracker.
-func (fetcher *DebianFetcher) FetchUpdate() (resp updater.FetcherResponse, err error) {
+func (fetcher *DebianFetcher) FetchUpdate(datastore database.Datastore) (resp updater.FetcherResponse, err error) {
 	log.Info("fetching Debian vulnerabilities")
 
 	// Download JSON.
@@ -68,7 +71,7 @@ func (fetcher *DebianFetcher) FetchUpdate() (resp updater.FetcherResponse, err e
 	}
 
 	// Get the SHA-1 of the latest update's JSON data
-	latestHash, err := database.GetFlagValue(debianUpdaterFlag)
+	latestHash, err := datastore.GetKeyValue(debianUpdaterFlag)
 	if err != nil {
 		return resp, err
 	}
@@ -103,7 +106,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F
 	err = json.NewDecoder(teedJSONReader).Decode(&data)
 	if err != nil {
 		log.Errorf("could not unmarshal Debian's JSON: %s", err)
-		return resp, ErrCouldNotParse
+		return resp, cerrors.ErrCouldNotParse
 	}
 
 	// Calculate the hash and skip updating if the hash has been seen before.
@@ -115,7 +118,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F
 
 	// Extract vulnerability data from Debian's JSON schema.
 	var unknownReleases map[string]struct{}
-	resp.Vulnerabilities, resp.Packages, unknownReleases = parseDebianJSON(&data)
+	resp.Vulnerabilities, unknownReleases = parseDebianJSON(&data)
 
 	// Log unknown releases
 	for k := range unknownReleases {
@@ -127,7 +130,7 @@ func buildResponse(jsonReader io.Reader, latestKnownHash string) (resp updater.F
 	return resp, nil
 }
 
-func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability, packages []*database.Package, unknownReleases map[string]struct{}) {
+func parseDebianJSON(data *jsonData) (vulnerabilities []database.Vulnerability, unknownReleases map[string]struct{}) {
 	mvulnerabilities := make(map[string]*database.Vulnerability)
 	unknownReleases = make(map[string]struct{})
 
@@ -140,34 +143,40 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability,
 					continue
 				}
 
-				// Skip if the release is not affected.
-				if releaseNode.FixedVersion == "0" || releaseNode.Status == "undetermined" {
+				// Skip if the status is not determined.
+				if releaseNode.Status == "undetermined" {
 					continue
 				}
 
 				// Get or create the vulnerability.
-				vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[vulnName]
+				namespaceName := "debian:" + database.DebianReleasesMapping[releaseName]
+				index := namespaceName + ":" + vulnName
+				vulnerability, vulnerabilityAlreadyExists := mvulnerabilities[index]
 				if !vulnerabilityAlreadyExists {
 					vulnerability = &database.Vulnerability{
-						ID:          vulnName,
+						Name:        vulnName,
+						Namespace:   database.Namespace{Name: namespaceName},
 						Link:        strings.Join([]string{cveURLPrefix, "/", vulnName}, ""),
-						Priority:    types.Unknown,
+						Severity:    types.Unknown,
 						Description: vulnNode.Description,
 					}
 				}
 
 				// Set the priority of the vulnerability.
 				// In the JSON, a vulnerability has one urgency per package it affects.
 				// The highest urgency should be the one set.
-				urgency := urgencyToPriority(releaseNode.Urgency)
-				if urgency.Compare(vulnerability.Priority) > 0 {
-					vulnerability.Priority = urgency
+				urgency := urgencyToSeverity(releaseNode.Urgency)
+				if urgency.Compare(vulnerability.Severity) > 0 {
+					vulnerability.Severity = urgency
 				}
 
 				// Determine the version of the package the vulnerability affects.
 				var version types.Version
 				var err error
-				if releaseNode.Status == "open" {
+				if releaseNode.FixedVersion == "0" {
+					// This means that the package is not affected by this vulnerability.
+					version = types.MinVersion
+				} else if releaseNode.Status == "open" {
 					// Open means that the package is currently vulnerable in the latest
 					// version of this Debian release.
 					version = types.MaxVersion
@@ -181,30 +190,28 @@ func parseDebianJSON(data *jsonData) (vulnerabilities []*database.Vulnerability,
 					}
 				}
 
-				// Create and add the package.
-				pkg := &database.Package{
-					OS:      "debian:" + database.DebianReleasesMapping[releaseName],
-					Name:    pkgName,
+				// Create and add the feature version.
+				pkg := database.FeatureVersion{
+					Feature: database.Feature{Name: pkgName},
 					Version: version,
 				}
-				vulnerability.FixedInNodes = append(vulnerability.FixedInNodes, pkg.GetNode())
-				packages = append(packages, pkg)
+				vulnerability.FixedIn = append(vulnerability.FixedIn, pkg)
 
 				// Store the vulnerability.
-				mvulnerabilities[vulnName] = vulnerability
+				mvulnerabilities[index] = vulnerability
 			}
 		}
 	}
 
 	// Convert the vulnerabilities map to a slice
 	for _, v := range mvulnerabilities {
-		vulnerabilities = append(vulnerabilities, v)
+		vulnerabilities = append(vulnerabilities, *v)
 	}
 
 	return
 }
 
-func urgencyToPriority(urgency string) types.Priority {
+func urgencyToSeverity(urgency string) types.Priority {
 	switch urgency {
 	case "not yet assigned":
 		return types.Unknown