database: add initial vulnerability support

database/database.go

@@ -28,8 +28,9 @@ type Datastore interface {
 	DeleteLayer(name string) error
 
 	// Vulnerability
-	// InsertVulnerabilities([]*Vulnerability)
-	// DeleteVulnerability(id string)
+	InsertVulnerabilities([]Vulnerability) error
+	// DeleteVulnerability(id string) error
+	FindVulnerability(namespaceName, name string) (Vulnerability, error)
 
 	// Notifications
 	// InsertNotifications([]Notification) error

database/models.go

@@ -2,6 +2,7 @@ package database
 
 import "github.com/coreos/clair/utils/types"
 
+// ID is only meant to be used by database implementations and should never be used for anything else.
 type Model struct {
 	ID int
 }
@@ -46,8 +47,9 @@ type Vulnerability struct {
 	Description string
 	Link        string
 	Severity    types.Priority
-	// FixedIn     map[types.Version]Feature // <<-- WRONG.
-	Affects []FeatureVersion
+
+	FixedIn []FeatureVersion
+	//Affects []FeatureVersion
 
 	// For output purposes. Only make sense when the vulnerability
 	// is already about a specific Feature/FeatureVersion.

database/pgsql/feature.go

@@ -1,6 +1,8 @@
 package pgsql
 
 import (
+	"database/sql"
+
 	"github.com/coreos/clair/database"
 	cerrors "github.com/coreos/clair/utils/errors"
 	"github.com/coreos/clair/utils/types"
@@ -54,6 +56,7 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
 	if err != nil {
 		return 0, err
 	}
+	featureVersion.Feature.ID = featureID
 
 	// Begin transaction.
 	tx, err := pgSQL.Begin()
@@ -62,6 +65,15 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
 		return 0, handleError("insertFeatureVersion.Begin()", err)
 	}
 
+	// Set transaction as SERIALIZABLE.
+	// This is how we ensure that the data in Vulnerability_Affects_FeatureVersion is always
+	// consistent.
+	_, err = tx.Exec(getQuery("set_tx_serializable"))
+	if err != nil {
+		tx.Rollback()
+		return 0, handleError("insertFeatureVersion.set_tx_serializable", err)
+	}
+
 	// Find or create FeatureVersion.
 	var newOrExisting string
 	err = tx.QueryRow(getQuery("soi_featureversion"), featureID, &featureVersion.Version).
@@ -77,20 +89,48 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
 
 	// Link the new FeatureVersion with every vulnerabilities that affect it, by inserting in
 	// Vulnerability_Affects_FeatureVersion.
+	err = linkFeatureVersionToVulnerabilities(tx, featureVersion)
+	if err != nil {
+		// tx.Rollback() is done in linkFeatureVersionToVulnerabilities.
+		return 0, err
+	}
 
-	// Lock Vulnerability_FixedIn_Feature because we can't let it to be modified while we modify
-	// Vulnerability_Affects_FeatureVersion.
-	_, err = tx.Exec(getQuery("l_share_vulnerability_fixedin_feature"))
+	// Commit transaction.
+	err = tx.Commit()
 	if err != nil {
-		tx.Rollback()
-		return 0, handleError("l_share_vulnerability_fixedin_feature", err)
+		return 0, handleError("insertFeatureVersion.Commit()", err)
+	}
+
+	if pgSQL.cache != nil {
+		pgSQL.cache.Add("featureversion:"+featureVersion.Feature.Name+":"+
+			featureVersion.Version.String(), featureVersion.ID)
 	}
 
+	return featureVersion.ID, nil
+}
+
+// TODO(Quentin-M): Batch me
+func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) {
+	IDs := make([]int, 0, len(featureVersions))
+
+	for i := 0; i < len(featureVersions); i++ {
+		id, err := pgSQL.insertFeatureVersion(featureVersions[i])
+		if err != nil {
+			return IDs, err
+		}
+		IDs = append(IDs, id)
+	}
+
+	return IDs, nil
+}
+
+func linkFeatureVersionToVulnerabilities(tx *sql.Tx, featureVersion database.FeatureVersion) error {
 	// Select every vulnerability and the fixed version that affect this Feature.
-	rows, err := tx.Query(getQuery("s_vulnerability_fixedin_feature"), featureID)
+	// TODO(Quentin-M): LIMIT
+	rows, err := tx.Query(getQuery("s_vulnerability_fixedin_feature"), featureVersion.Feature.ID)
 	if err != nil {
 		tx.Rollback()
-		return 0, handleError("s_vulnerability_fixedin_feature", err)
+		return handleError("s_vulnerability_fixedin_feature", err)
 	}
 	defer rows.Close()
 
@@ -100,7 +140,7 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
 		err := rows.Scan(&fixedInID, &vulnerabilityID, &fixedInVersion)
 		if err != nil {
 			tx.Rollback()
-			return 0, handleError("s_vulnerability_fixedin_feature.Scan()", err)
+			return handleError("s_vulnerability_fixedin_feature.Scan()", err)
 		}
 
 		if featureVersion.Version.Compare(fixedInVersion) < 0 {
@@ -111,40 +151,14 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
 				featureVersion.ID, fixedInID)
 			if err != nil {
 				tx.Rollback()
-				return 0, handleError("i_vulnerability_affects_featureversion", err)
+				return handleError("i_vulnerability_affects_featureversion", err)
 			}
 		}
 	}
 	if err = rows.Err(); err != nil {
-		return 0, handleError("s_vulnerability_fixedin_feature.Rows()", err)
-	}
-
-	// Commit transaction.
-	err = tx.Commit()
-	if err != nil {
 		tx.Rollback()
-		return 0, handleError("insertFeatureVersion.Commit()", err)
-	}
-
-	if pgSQL.cache != nil {
-		pgSQL.cache.Add("featureversion:"+featureVersion.Feature.Name+":"+
-			featureVersion.Version.String(), featureVersion.ID)
+		return handleError("s_vulnerability_fixedin_feature.Rows()", err)
 	}
 
-	return featureVersion.ID, nil
-}
-
-// TODO(Quentin-M): Batch me
-func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) {
-	IDs := make([]int, 0, len(featureVersions))
-
-	for i := 0; i < len(featureVersions); i++ {
-		id, err := pgSQL.insertFeatureVersion(featureVersions[i])
-		if err != nil {
-			return IDs, err
-		}
-		IDs = append(IDs, id)
-	}
-
-	return IDs, nil
+	return nil
 }

database/pgsql/layer.go

@@ -77,7 +77,7 @@ func (pgSQL *pgSQL) getLayerFeatureVersions(layerID int, idOnly bool) ([]databas
 
 	// Query
 	rows, err := pgSQL.Query(query, layerID)
-	if err != nil && err != sql.ErrNoRows {
+	if err != nil {
 		return featureVersions, handleError(query, err)
 	}
 	defer rows.Close()
@@ -201,22 +201,30 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
 	if err != nil && err != cerrors.ErrNotFound {
 		return err
 	} else if err == nil {
+		if existingLayer.EngineVersion >= layer.EngineVersion {
+			// The layer exists and has an equal or higher engine verison, do nothing.
+			return nil
+		}
+
 		layer.ID = existingLayer.ID
 	}
 
-	// Begin transaction.
-	tx, err := pgSQL.Begin()
-	if err != nil {
-		tx.Rollback()
-		return handleError("InsertLayer.Begin()", err)
+	// Get parent ID.
+	var parentID zero.Int
+	if layer.Parent != nil {
+		if layer.Parent.ID == 0 {
+			log.Warning("Parent is expected to be retrieved from database when inserting a layer.")
+			return cerrors.NewBadRequestError("Parent is expected to be retrieved from database when inserting a layer.")
+		}
+
+		parentID = zero.IntFrom(int64(layer.Parent.ID))
 	}
 
 	// Find or insert namespace if provided.
 	var namespaceID zero.Int
 	if layer.Namespace != nil {
 		n, err := pgSQL.insertNamespace(*layer.Namespace)
 		if err != nil {
-			tx.Rollback()
 			return err
 		}
 		namespaceID = zero.IntFrom(int64(n))
@@ -227,30 +235,22 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
 		}
 	}
 
+	// Begin transaction.
+	tx, err := pgSQL.Begin()
+	if err != nil {
+		tx.Rollback()
+		return handleError("InsertLayer.Begin()", err)
+	}
+
 	if layer.ID == 0 {
 		// Insert a new layer.
-		var parentID zero.Int
-		if layer.Parent != nil {
-			if layer.Parent.ID == 0 {
-				log.Warning("Parent is expected to be retrieved from database when inserting a layer.")
-				return cerrors.NewBadRequestError("Parent is expected to be retrieved from database when inserting a layer.")
-			}
-
-			parentID = zero.IntFrom(int64(layer.Parent.ID))
-		}
-
 		err = tx.QueryRow(getQuery("i_layer"), layer.Name, layer.EngineVersion, parentID, namespaceID).
 			Scan(&layer.ID)
 		if err != nil {
 			tx.Rollback()
 			return handleError("i_layer", err)
 		}
 	} else {
-		if existingLayer.EngineVersion >= layer.EngineVersion {
-			// The layer exists and has an equal or higher engine verison, do nothing.
-			return nil
-		}
-
 		// Update an existing layer.
 		_, err = tx.Exec(getQuery("u_layer"), layer.ID, layer.EngineVersion, namespaceID)
 		if err != nil {
@@ -269,6 +269,7 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
 	// Update Layer_diff_FeatureVersion now.
 	err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
 	if err != nil {
+		tx.Rollback()
 		return err
 	}
 
@@ -293,7 +294,7 @@ func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *
 	} else if layer.Parent != nil {
 		// There is a parent, we need to diff the Features with it.
 
-		// Build name:version strctures.
+		// Build name:version structures.
 		layerFeaturesMapNV, layerFeaturesNV := createNV(layer.Features)
 		parentLayerFeaturesMapNV, parentLayerFeaturesNV := createNV(layer.Parent.Features)
 

database/pgsql/migrations/20151222113213_Initial.sql

@@ -73,7 +73,7 @@ CREATE TABLE IF NOT EXISTS Vulnerability (
   name VARCHAR(128) NOT NULL,
   description TEXT NULL,
   link VARCHAR(128) NULL,
-  severity severity NULL,
+  severity severity NOT NULL,
 
   UNIQUE (namespace_id, name));
 
@@ -137,6 +137,6 @@ DROP TABLE IF EXISTS Namespace,
                      Vulnerability,
                      Vulnerability_FixedIn_Feature,
                      Vulnerability_Affects_FeatureVersion,
-                     KeyValue
+                     KeyValue,
                      Lock
             CASCADE;

database/pgsql/pgsql.go

@@ -120,7 +120,7 @@ func dropDatabase(dataSource, databaseName string) error {
 	// Drop database.
 	_, err = db.Exec("DROP DATABASE " + databaseName + ";")
 	if err != nil {
-		return fmt.Errorf("could not create database: %v", err)
+		return fmt.Errorf("could not drop database: %v", err)
 	}
 
 	return nil
@@ -185,7 +185,7 @@ func handleError(desc string, err error) error {
 		return database.ErrBackendException
 	} else if err == sql.ErrNoRows {
 		return cerrors.ErrNotFound
-	} else if err == sql.ErrTxDone {
+	} else if err == sql.ErrTxDone || strings.HasPrefix(err.Error(), "sql:") {
 		return database.ErrBackendException
 	}
 

database/pgsql/queries.go

@@ -10,6 +10,8 @@ var queries map[string]string
 func init() {
 	queries = make(map[string]string)
 
+	queries["set_tx_serializable"] = `SET TRANSACTION ISOLATION LEVEL SERIALIZABLE`
+
 	// keyvalue.go
 	queries["u_keyvalue"] = `UPDATE KeyValue SET value = $1 WHERE key = $2`
 	queries["i_keyvalue"] = `INSERT INTO KeyValue(key, value) VALUES($1, $2)`
@@ -39,10 +41,6 @@ func init() {
     UNION
     SELECT id FROM new_feature`
 
-	queries["l_share_vulnerability_fixedin_feature"] = `
-    LOCK Vulnerability_FixedIn_Feature IN SHARE MODE
-  `
-
 	queries["soi_featureversion"] = `
     WITH new_featureversion AS (
       INSERT INTO FeatureVersion(feature_id, version)
@@ -142,6 +140,41 @@ func init() {
 	queries["r_lock"] = `DELETE FROM Lock WHERE name = $1 AND owner = $2`
 
 	queries["r_lock_expired"] = `DELETE FROM LOCK WHERE until < CURRENT_TIMESTAMP`
+
+	// vulnerability.go
+	queries["f_vulnerability"] = `
+    SELECT v.id, n.id, v.description, v.link, v.severity, vfif.version, f.id, f.Name
+    FROM Vulnerability v
+      JOIN Namespace n ON v.namespace_id = n.id
+      LEFT JOIN Vulnerability_FixedIn_Feature vfif ON v.id = vfif.vulnerability_id
+      LEFT JOIN Feature f ON vfif.feature_id = f.id
+    WHERE n.Name = $1 AND v.Name = $2`
+
+	queries["i_vulnerability"] = `
+    INSERT INTO Vulnerability(namespace_id, name, description, link, severity)
+    VALUES($1, $2, $3, $4, $5)
+    RETURNING id`
+
+	queries["u_vulnerability"] = `
+    UPDATE Vulnerability SET description = $2, link = $3, severity = $4 WHERE id = $1`
+
+	queries["i_vulnerability_fixedin_feature"] = `
+    INSERT INTO Vulnerability_FixedIn_Feature(vulnerability_id, feature_id, version)
+    VALUES($1, $2, $3)
+    RETURNING id`
+
+	queries["u_vulnerability_fixedin_feature"] = `
+    UPDATE Vulnerability_FixedIn_Feature
+    SET version = $3
+    WHERE vulnerability_id = $1 AND feature_id = $2
+    RETURNING id`
+
+	queries["r_vulnerability_affects_featureversion"] = `
+    DELETE FROM Vulnerability_Affects_FeatureVersion
+    WHERE fixedin_id = $1`
+
+	queries["f_featureversion_by_feature"] = `
+    SELECT id, version FROM FeatureVersion WHERE feature_id = $1`
 }
 
 func getQuery(name string) string {

database/pgsql/testdata/data.sql

@@ -3,6 +3,7 @@ INSERT INTO namespace (id, name) VALUES (2, 'debian:8');
 
 INSERT INTO feature (id, namespace_id, name) VALUES (1, 1, 'wechat');
 INSERT INTO feature (id, namespace_id, name) VALUES (2, 1, 'openssl');
+INSERT INTO feature (id, namespace_id, name) VALUES (4, 1, 'libssl');
 INSERT INTO feature (id, namespace_id, name) VALUES (3, 2, 'openssl');
 INSERT INTO featureversion (id, feature_id, version) VALUES (1, 1, '0.5');
 INSERT INTO featureversion (id, feature_id, version) VALUES (2, 2, '1.0');
@@ -23,8 +24,9 @@ INSERT INTO layer_diff_featureversion (id, layer_id, featureversion_id, modifica
 
 INSERT INTO vulnerability (id, namespace_id, name, description, link, severity) VALUES (1, 1, 'CVE-OPENSSL-1-DEB7', 'A vulnerability affecting OpenSSL < 2.0 on Debian 7.0', 'http://google.com/#q=CVE-OPENSSL-1-DEB7', 'High');
 INSERT INTO vulnerability_fixedin_feature (id, vulnerability_id, feature_id, version) VALUES (1, 1, 2, '2.0');
+INSERT INTO vulnerability_fixedin_feature (id, vulnerability_id, feature_id, version) VALUES (2, 1, 4, '1.9-abc');
 INSERT INTO vulnerability_affects_featureversion (id, vulnerability_id, featureversion_id, fixedin_id) VALUES (1, 1, 2, 1); -- CVE-OPENSSL-1-DEB7 affects Debian:7 OpenSSL 1.0
-INSERT INTO vulnerability (id, namespace_id, name, description, link, severity) VALUES (2, 1, 'CVE-NOPE', 'A vulnerability affecting nothing', 'http://google.com/#q=NOPE', 'Negligible');
+INSERT INTO vulnerability (id, namespace_id, name, description, link, severity) VALUES (2, 1, 'CVE-NOPE', 'A vulnerability affecting nothing', '', 'Unknown');
 
 SELECT pg_catalog.setval(pg_get_serial_sequence('namespace', 'id'), (SELECT MAX(id) FROM namespace)+1);
 SELECT pg_catalog.setval(pg_get_serial_sequence('feature', 'id'), (SELECT MAX(id) FROM feature)+1);