Skip to content

Commit

Permalink
database: do insert/find layers (with their features and vulnerabilit…
Browse files Browse the repository at this point in the history 
…ies)
  • Loading branch information
@Quentin-M @jzelinskie
Quentin-M authored and jzelinskie committed Feb 24, 2016
1 parent 2c150b0 commit 970756c
Show file tree
Hide file tree
Showing 6 changed files with 195 additions and 69 deletions.
56 changes: 38 additions & 18 deletions database/pgsql/feature.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ package pgsql

import (
"github.com/coreos/clair/database"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/coreos/clair/utils/types"
cerrors "github.com/coreos/clair/utils/errors"
)

func (pgSQL *pgSQL) insertFeature(feature database.Feature) (id int, err error) {
func (pgSQL *pgSQL) insertFeature(feature database.Feature) (int, error) {
if feature.Name == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid Feature")
}
Expand All @@ -20,25 +20,29 @@ func (pgSQL *pgSQL) insertFeature(feature database.Feature) (id int, err error)
// Find or create Namespace.
namespaceID, err := pgSQL.insertNamespace(feature.Namespace)
if err != nil {
return -1, err
return 0, err
}

// Find or create Feature.
var id int
err = pgSQL.QueryRow(getQuery("soi_feature"), feature.Name, namespaceID).Scan(&id)
if err != nil {
return 0, err
}

if pgSQL.cache != nil {
pgSQL.cache.Add("feature:"+feature.Name, id)
}

return
return id, nil
}

func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion) (id int, err error) {
if featureVersion.Version.String() == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion")
}
if featureVersion.Version.String() == "" {
return 0, cerrors.NewBadRequestError("could not find/insert invalid FeatureVersion")
}

if pgSQL.cache != nil {
if pgSQL.cache != nil {
if id, found := pgSQL.cache.Get("featureversion:" + featureVersion.Feature.Name + ":" +
featureVersion.Version.String()); found {
return id.(int), nil
Expand All @@ -48,23 +52,23 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
// Find or create Feature first.
featureID, err := pgSQL.insertFeature(featureVersion.Feature)
if err != nil {
return -1, err
return 0, err
}

// Begin transaction.
tx, err := pgSQL.Begin()
if err != nil {
tx.Rollback()
return -1, err
return 0, err
}

// Find or create FeatureVersion.
var newOrExisting string
err = tx.QueryRow(getQuery("soi_featureversion"), featureID, featureVersion.Version).
err = tx.QueryRow(getQuery("soi_featureversion"), featureID, &featureVersion.Version).
Scan(&newOrExisting, &featureVersion.ID)
if err != nil {
tx.Rollback()
return -1, err
return 0, err
}
if newOrExisting == "exi" {
// That featureVersion already exists, return its id.
Expand All @@ -79,14 +83,14 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
_, err = tx.Exec(getQuery("l_share_vulnerability_fixedin_feature"))
if err != nil {
tx.Rollback()
return -1, err
return 0, err
}

// Select every vulnerability and the fixed version that affect this Feature.
rows, err := tx.Query(getQuery("s_vulnerability_fixedin_feature"), featureID)
if err != nil {
tx.Rollback()
return -1, err
return 0, err
}
defer rows.Close()

Expand All @@ -96,17 +100,18 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
err := rows.Scan(&fixedInID, &vulnerabilityID, &fixedInVersion)
if err != nil {
tx.Rollback()
return -1, err
return 0, err
}

if featureVersion.Version.Compare(fixedInVersion) < 0 {
// The version of the FeatureVersion we are inserting is lower than the fixed version on this
// Vulnerability, thus, this FeatureVersion is affected by it.
// TODO(Quentin-M): Prepare.
_, err := tx.Exec(getQuery("i_vulnerability_affects_featureversion"), vulnerabilityID,
featureVersion.ID, fixedInID)
if err != nil {
tx.Rollback()
return -1, err
return 0, err
}
}
}
Expand All @@ -115,13 +120,28 @@ func (pgSQL *pgSQL) insertFeatureVersion(featureVersion database.FeatureVersion)
err = tx.Commit()
if err != nil {
tx.Rollback()
return -1, err
return 0, err
}

if pgSQL.cache != nil {
pgSQL.cache.Add("featureversion:"+featureVersion.Feature.Name+":"+
featureVersion.Version.String(), featureVersion.ID)
}

return 0, nil
return featureVersion.ID, nil
}

// TODO(Quentin-M): Batch me
func (pgSQL *pgSQL) insertFeatureVersions(featureVersions []database.FeatureVersion) ([]int, error) {
IDs := make([]int, 0, len(featureVersions))

for i := 0; i < len(featureVersions); i++ {
id, err := pgSQL.insertFeatureVersion(featureVersions[i])
if err != nil {
return IDs, err
}
IDs = append(IDs, id)
}

return IDs, nil
}
112 changes: 93 additions & 19 deletions database/pgsql/layer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,22 @@ import (
"database/sql"

"github.com/coreos/clair/database"
"github.com/coreos/clair/utils"
cerrors "github.com/coreos/clair/utils/errors"
"github.com/guregu/null/zero"
)

func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities bool) (database.Layer, error) {
// Find the layer
var layer database.Layer
var parentName sql.NullString
var parentID zero.Int
var parentName zero.String
var namespaceID zero.Int
var namespaceName sql.NullString

err := pgSQL.QueryRow(getQuery("s_layer"), name).
Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentName, &namespaceName)
Scan(&layer.ID, &layer.Name, &layer.EngineVersion, &parentID, &parentName, &namespaceID,
&namespaceName)

if err == sql.ErrNoRows {
return layer, cerrors.ErrNotFound
Expand All @@ -24,11 +28,17 @@ func (pgSQL *pgSQL) FindLayer(name string, withFeatures, withVulnerabilities boo
return layer, err
}

if parentName.Valid {
layer.Parent = &database.Layer{Name: parentName.String}
if !parentID.IsZero() {
layer.Parent = &database.Layer{
Model: database.Model{ID: int(parentID.Int64)},
Name: parentName.String,
}
}
if namespaceName.Valid {
layer.Namespace = &database.Namespace{Name: namespaceName.String}
if !namespaceID.IsZero() {
layer.Namespace = &database.Namespace{
Model: database.Model{ID: int(namespaceID.Int64)},
Name: namespaceName.String,
}
}

// Find its features
Expand Down Expand Up @@ -210,6 +220,11 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
return err
}
namespaceID = zero.IntFrom(int64(n))
} else if layer.Namespace == nil && layer.Parent != nil {
// Import the Namespace from the parent if it has one and this layer doesn't specify one.
if layer.Parent.Namespace != nil {
namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
}
}

if layer.ID == 0 {
Expand All @@ -222,11 +237,6 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
}

parentID = zero.IntFrom(int64(layer.Parent.ID))

// Import the Namespace from the parent is this layer doesn't specify one.
if zero.IsNull(namespaceID) {
namespaceID = zero.IntFrom(int64(layer.Parent.Namespace.ID))
}
}

err = tx.QueryRow(getQuery("i_layer"), layer.Name, layer.EngineVersion, parentID, namespaceID).
Expand All @@ -247,10 +257,20 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
tx.Rollback()
return err
}

// Remove all existing Layer_diff_FeatureVersion.
_, err = tx.Exec(getQuery("r_layer_diff_featureversion"), layer.ID)
if err != nil {
tx.Rollback()
return err
}
}

// Update Layer_diff_FeatureVersion now.
updateDiffFeatureVersions(tx, &layer, &existingLayer)
err = pgSQL.updateDiffFeatureVersions(tx, &layer, &existingLayer)
if err != nil {
return err
}

// Commit transaction.
err = tx.Commit()
Expand All @@ -262,19 +282,73 @@ func (pgSQL *pgSQL) InsertLayer(layer database.Layer) error {
return nil
}

func updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *database.Layer) {
// TODO

if existingLayer != nil {
// We are updating a layer, we need to diff the Features with the existing Layer.
func (pgSQL *pgSQL) updateDiffFeatureVersions(tx *sql.Tx, layer, existingLayer *database.Layer) error {
// add and del are the FeatureVersion diff we should insert.
var add []database.FeatureVersion
var del []database.FeatureVersion

} else if layer.Parent == nil {
if layer.Parent == nil {
// There is no parent, every Features are added.

add = append(add, layer.Features...)
} else if layer.Parent != nil {
// There is a parent, we need to diff the Features with it.

// Build name:version strctures.
layerFeaturesMapNV, layerFeaturesNV := createNV(layer.Features)
parentLayerFeaturesMapNV, parentLayerFeaturesNV := createNV(layer.Parent.Features)

// Calculate the added and deleted FeatureVersions name:version.
addNV := utils.CompareStringLists(layerFeaturesNV, parentLayerFeaturesNV)
delNV := utils.CompareStringLists(parentLayerFeaturesNV, layerFeaturesNV)

// Fill the structures containing the added and deleted FeatureVersions
for _, nv := range addNV {
add = append(add, *layerFeaturesMapNV[nv])
}
for _, nv := range delNV {
del = append(del, *parentLayerFeaturesMapNV[nv])
}
}

// Insert FeatureVersions in the database.
addIDs, err := pgSQL.insertFeatureVersions(add)
if err != nil {
return err
}
delIDs, err := pgSQL.insertFeatureVersions(del)
if err != nil {
return err
}

// Insert diff in the database.
if len(addIDs) > 0 {
_, err = tx.Exec(getQuery("i_layer_diff_featureversion"), layer.ID, "add", buildInputArray(addIDs))
if err != nil {
return err
}
}
if len(delIDs) > 0 {
_, err = tx.Exec(getQuery("i_layer_diff_featureversion"), layer.ID, "del", buildInputArray(delIDs))
if err != nil {
return err
}
}

return nil
}

func createNV(features []database.FeatureVersion) (map[string]*database.FeatureVersion, []string) {
mapNV := make(map[string]*database.FeatureVersion, 0)
sliceNV := make([]string, 0, len(features))

for i := 0; i < len(features); i++ {
featureVersion := &features[i]
nv := featureVersion.Feature.Name + ":" + featureVersion.Version.String()
mapNV[nv] = featureVersion
sliceNV = append(sliceNV, nv)
}

return mapNV, sliceNV
}

func (pgSQL *pgSQL) DeleteLayer(name string) error {
Expand Down
23 changes: 15 additions & 8 deletions database/pgsql/layer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,8 @@ func testInsertLayerInvalid(t *testing.T, datastore database.Datastore) {
}

func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
fmt.Println("- testInsertLayerTree")

f1 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace2"},
Expand Down Expand Up @@ -152,7 +154,7 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
f5 := database.FeatureVersion{
Feature: database.Feature{
Namespace: database.Namespace{Name: "TestInsertLayerNamespace3"},
Name: "TestInsertLayerFeature2",
Name: "TestInsertLayerFeature3",
},
Version: types.NewVersionUnsafe("0.57"),
}
Expand Down Expand Up @@ -180,10 +182,11 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
Namespace: &database.Namespace{Name: "TestInsertLayerNamespace2"},
Features: []database.FeatureVersion{f1, f2, f3},
},
// This layer covers the case where the last layer doesn't provide any Feature.
// This layer covers the case where the last layer doesn't provide any new Feature.
database.Layer{
Name: "TestInsertLayer4a",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Name: "TestInsertLayer4a",
Parent: &database.Layer{Name: "TestInsertLayer3"},
Features: []database.FeatureVersion{f1, f2, f3},
},
// This layer covers the case where the last layer provides Features.
// It also modifies the Namespace ("upgrade") but keeps some Features not upgraded, their
Expand Down Expand Up @@ -221,7 +224,9 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
}

l4a := retrievedLayers["TestInsertLayer4a"]
assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name)
if assert.NotNil(t, l4a.Namespace) {
assert.Equal(t, "TestInsertLayerNamespace2", l4a.Namespace.Name)
}
assert.Len(t, l4a.Features, 3)
for _, featureVersion := range l4a.Features {
if cmpFV(featureVersion, f1) && cmpFV(featureVersion, f2) && cmpFV(featureVersion, f3) {
Expand All @@ -230,9 +235,11 @@ func testInsertLayerTree(t *testing.T, datastore database.Datastore) {
}

l4b := retrievedLayers["TestInsertLayer4b"]
assert.Equal(t, "TestInsertLayerNamespace3", l4a.Namespace.Name)
assert.Len(t, l4a.Features, 3)
for _, featureVersion := range l4a.Features {
if assert.NotNil(t, l4b.Namespace) {
assert.Equal(t, "TestInsertLayerNamespace3", l4b.Namespace.Name)
}
assert.Len(t, l4b.Features, 3)
for _, featureVersion := range l4b.Features {
if cmpFV(featureVersion, f2) && cmpFV(featureVersion, f5) && cmpFV(featureVersion, f6) {
assert.Error(t, fmt.Errorf("TestInsertLayer4a contains an unexpected package: %#v. Should contain %#v and %#v and %#v.", featureVersion, f2, f4, f6))
}
Expand Down

0 comments on commit 970756c

Please sign in to comment.