Skip to content

Commit

Permalink
pgsql: Fix postgres queries for feature_type
Browse files Browse the repository at this point in the history
  • Loading branch information
@KeyboardNerd
KeyboardNerd committed Feb 19, 2019
1 parent 5fa1ac8 commit 79af05e
Show file tree
Hide file tree
Showing 12 changed files with 197 additions and 272 deletions.
6 changes: 4 additions & 2 deletions database/pgsql/ancestry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,11 @@ const (

findAncestryFeatures = `
SELECT namespace.name, namespace.version_format, feature.name,
feature.version, feature.version_format, ancestry_layer.ancestry_index,
feature.version, feature.version_format, feature_type.name, ancestry_layer.ancestry_index,
ancestry_feature.feature_detector_id, ancestry_feature.namespace_detector_id
FROM namespace, feature, namespaced_feature, ancestry_layer, ancestry_feature
FROM namespace, feature, feature_type, namespaced_feature, ancestry_layer, ancestry_feature
WHERE ancestry_layer.ancestry_id = $1
AND feature_type.id = feature.type
AND ancestry_feature.ancestry_layer_id = ancestry_layer.id
AND ancestry_feature.namespaced_feature_id = namespaced_feature.id
AND namespaced_feature.feature_id = feature.id
Expand Down Expand Up @@ -256,6 +257,7 @@ func (tx *pgSession) findAncestryFeatures(ancestryID int64, detectors detectorMa
&feature.Feature.Name,
&feature.Feature.Version,
&feature.Feature.VersionFormat,
&feature.Feature.Type,
&index,
&featureDetectorID,
&namespaceDetectorID,
Expand Down
75 changes: 22 additions & 53 deletions database/pgsql/complex_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

"github.com/pborman/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/coreos/clair/database"
"github.com/coreos/clair/ext/versionfmt"
Expand Down Expand Up @@ -65,19 +66,13 @@ func testGenRandomVulnerabilityAndNamespacedFeature(t *testing.T, store database
for i := 0; i < numFeatures; i++ {
version := rand.Intn(numFeatures)

features[i] = database.Feature{
Name: featureName,
VersionFormat: featureVersionFormat,
Version: strconv.Itoa(version),
}

features[i] = *database.NewSourcePackage(featureName, strconv.Itoa(version), featureVersionFormat)
nsFeatures[i] = database.NamespacedFeature{
Namespace: namespace,
Feature: features[i],
}
}

// insert features
if !assert.Nil(t, tx.PersistFeatures(features)) {
t.FailNow()
}
Expand All @@ -98,6 +93,7 @@ func testGenRandomVulnerabilityAndNamespacedFeature(t *testing.T, store database
{
Namespace: namespace,
FeatureName: featureName,
FeatureType: database.SourcePackage,
AffectedVersion: strconv.Itoa(version),
FixedInVersion: strconv.Itoa(version),
},
Expand All @@ -117,7 +113,6 @@ func TestConcurrency(t *testing.T) {
t.FailNow()
}
defer store.Close()

start := time.Now()
var wg sync.WaitGroup
wg.Add(100)
Expand All @@ -137,65 +132,39 @@ func TestConcurrency(t *testing.T) {
fmt.Println("total", time.Since(start))
}

func genRandomNamespaces(t *testing.T, count int) []database.Namespace {
r := make([]database.Namespace, count)
for i := 0; i < count; i++ {
r[i] = database.Namespace{
Name: uuid.New(),
VersionFormat: "dpkg",
}
}
return r
}

func TestCaching(t *testing.T) {
store, err := openDatabaseForTest("Caching", false)
if !assert.Nil(t, err) {
t.FailNow()
}
defer store.Close()

nsFeatures, vulnerabilities := testGenRandomVulnerabilityAndNamespacedFeature(t, store)
tx, err := store.Begin()
if !assert.Nil(t, err) {
t.FailNow()
}

fmt.Printf("%d features, %d vulnerabilities are generated", len(nsFeatures), len(vulnerabilities))

var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
tx, err := store.Begin()
if !assert.Nil(t, err) {
t.FailNow()
}

assert.Nil(t, tx.PersistNamespacedFeatures(nsFeatures))
fmt.Println("finished to insert namespaced features")

tx.Commit()
}()

go func() {
defer wg.Done()
tx, err := store.Begin()
if !assert.Nil(t, err) {
t.FailNow()
}

assert.Nil(t, tx.InsertVulnerabilities(vulnerabilities))
fmt.Println("finished to insert vulnerabilities")
tx.Commit()
require.Nil(t, tx.PersistNamespacedFeatures(nsFeatures))
if err := tx.Commit(); err != nil {
panic(err)
}

}()
tx, err = store.Begin()
if !assert.Nil(t, err) {
t.FailNow()
}

wg.Wait()
require.Nil(t, tx.InsertVulnerabilities(vulnerabilities))
if err := tx.Commit(); err != nil {
panic(err)
}

tx, err := store.Begin()
tx, err = store.Begin()
if !assert.Nil(t, err) {
t.FailNow()
}
defer tx.Rollback()

// Verify consistency now.
affected, err := tx.FindAffectedNamespacedFeatures(nsFeatures)
if !assert.Nil(t, err) {
t.FailNow()
Expand All @@ -220,7 +189,7 @@ func TestCaching(t *testing.T) {
actualAffectedNames = append(actualAffectedNames, s.Name)
}

assert.Len(t, strutil.Difference(expectedAffectedNames, actualAffectedNames), 0)
assert.Len(t, strutil.Difference(actualAffectedNames, expectedAffectedNames), 0)
require.Len(t, strutil.Difference(expectedAffectedNames, actualAffectedNames), 0, "\nvulns: %#v\nfeature:%#v\nexpected:%#v\nactual:%#v", vulnerabilities, ansf.NamespacedFeature, expectedAffectedNames, actualAffectedNames)
require.Len(t, strutil.Difference(actualAffectedNames, expectedAffectedNames), 0)
}
}
94 changes: 46 additions & 48 deletions database/pgsql/feature.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ const (
AND nf.feature_id = f.id
AND nf.namespace_id = v.namespace_id
AND vaf.feature_name = f.name
AND vaf.feature_type = f.type
AND vaf.vulnerability_id = v.id
AND v.deleted_at IS NULL`

Expand All @@ -68,6 +69,11 @@ func (tx *pgSession) PersistFeatures(features []database.Feature) error {
return nil
}

types, err := tx.getFeatureTypeMap()
if err != nil {
return err
}

// Sorting is needed before inserting into database to prevent deadlock.
sort.Slice(features, func(i, j int) bool {
return features[i].Name < features[j].Name ||
Expand All @@ -78,13 +84,13 @@ func (tx *pgSession) PersistFeatures(features []database.Feature) error {
// TODO(Sida): A better interface for bulk insertion is needed.
keys := make([]interface{}, 0, len(features)*3)
for _, f := range features {
keys = append(keys, f.Name, f.Version, f.VersionFormat)
keys = append(keys, f.Name, f.Version, f.VersionFormat, types.byName[f.Type])
if f.Name == "" || f.Version == "" || f.VersionFormat == "" {
return commonerr.NewBadRequestError("Empty feature name, version or version format is not allowed")
}
}

_, err := tx.Exec(queryPersistFeature(len(features)), keys...)
_, err = tx.Exec(queryPersistFeature(len(features)), keys...)
return handleError("queryPersistFeature", err)
}

Expand Down Expand Up @@ -240,62 +246,39 @@ func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFea
return nil
}

// FindAffectedNamespacedFeatures looks up cache table and retrieves all
// vulnerabilities associated with the features.
// FindAffectedNamespacedFeatures retrieves vulnerabilities associated with the
// feature.
func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.NamespacedFeature) ([]database.NullableAffectedNamespacedFeature, error) {
if len(features) == 0 {
return nil, nil
}

returnFeatures := make([]database.NullableAffectedNamespacedFeature, len(features))

// featureMap is used to keep track of duplicated features.
featureMap := map[database.NamespacedFeature][]*database.NullableAffectedNamespacedFeature{}
// initialize return value and generate unique feature request queries.
for i, f := range features {
returnFeatures[i] = database.NullableAffectedNamespacedFeature{
AffectedNamespacedFeature: database.AffectedNamespacedFeature{
NamespacedFeature: f,
},
}

featureMap[f] = append(featureMap[f], &returnFeatures[i])
}

// query unique namespaced features
distinctFeatures := []database.NamespacedFeature{}
for f := range featureMap {
distinctFeatures = append(distinctFeatures, f)
}

nsFeatureIDs, err := tx.findNamespacedFeatureIDs(distinctFeatures)
vulnerableFeatures := make([]database.NullableAffectedNamespacedFeature, len(features))
featureIDs, err := tx.findNamespacedFeatureIDs(features)
if err != nil {
return nil, err
}

toQuery := []int64{}
featureIDMap := map[int64][]*database.NullableAffectedNamespacedFeature{}
for i, id := range nsFeatureIDs {
for i, id := range featureIDs {
if id.Valid {
toQuery = append(toQuery, id.Int64)
for _, f := range featureMap[distinctFeatures[i]] {
f.Valid = id.Valid
featureIDMap[id.Int64] = append(featureIDMap[id.Int64], f)
}
vulnerableFeatures[i].Valid = true
vulnerableFeatures[i].NamespacedFeature = features[i]
}
}

rows, err := tx.Query(searchNamespacedFeaturesVulnerabilities, pq.Array(toQuery))
rows, err := tx.Query(searchNamespacedFeaturesVulnerabilities, pq.Array(featureIDs))
if err != nil {
return nil, handleError("searchNamespacedFeaturesVulnerabilities", err)
}

defer rows.Close()

for rows.Next() {
var (
featureID int64
vuln database.VulnerabilityWithFixedIn
)

err := rows.Scan(&featureID,
&vuln.Name,
&vuln.Description,
Expand All @@ -306,28 +289,30 @@ func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.Namespac
&vuln.Namespace.Name,
&vuln.Namespace.VersionFormat,
)

if err != nil {
return nil, handleError("searchNamespacedFeaturesVulnerabilities", err)
}

for _, f := range featureIDMap[featureID] {
f.AffectedBy = append(f.AffectedBy, vuln)
for i, id := range featureIDs {
if id.Valid && id.Int64 == featureID {
vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy = append(vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy, vuln)
}
}
}

return returnFeatures, nil
return vulnerableFeatures, nil
}

func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) ([]sql.NullInt64, error) {
if len(nfs) == 0 {
return nil, nil
}

nfsMap := map[database.NamespacedFeature]sql.NullInt64{}
keys := make([]interface{}, 0, len(nfs)*4)
nfsMap := map[database.NamespacedFeature]int64{}
keys := make([]interface{}, 0, len(nfs)*5)
for _, nf := range nfs {
keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Namespace.Name)
nfsMap[nf] = sql.NullInt64{}
keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Type, nf.Namespace.Name)
}

rows, err := tx.Query(querySearchNamespacedFeature(len(nfs)), keys...)
Expand All @@ -337,12 +322,12 @@ func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature)

defer rows.Close()
var (
id sql.NullInt64
id int64
nf database.NamespacedFeature
)

for rows.Next() {
err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Namespace.Name)
err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Type, &nf.Namespace.Name)
nf.Namespace.VersionFormat = nf.VersionFormat
if err != nil {
return nil, handleError("searchNamespacedFeature", err)
Expand All @@ -352,7 +337,11 @@ func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature)

ids := make([]sql.NullInt64, len(nfs))
for i, nf := range nfs {
ids[i] = nfsMap[nf]
if id, ok := nfsMap[nf]; ok {
ids[i] = sql.NullInt64{id, true}
} else {
ids[i] = sql.NullInt64{}
}
}

return ids, nil
Expand All @@ -363,11 +352,17 @@ func (tx *pgSession) findFeatureIDs(fs []database.Feature) ([]sql.NullInt64, err
return nil, nil
}

types, err := tx.getFeatureTypeMap()
if err != nil {
return nil, err
}

fMap := map[database.Feature]sql.NullInt64{}

keys := make([]interface{}, 0, len(fs)*3)
keys := make([]interface{}, 0, len(fs)*4)
for _, f := range fs {
keys = append(keys, f.Name, f.Version, f.VersionFormat)
typeID := types.byName[f.Type]
keys = append(keys, f.Name, f.Version, f.VersionFormat, typeID)
fMap[f] = sql.NullInt64{}
}

Expand All @@ -382,10 +377,13 @@ func (tx *pgSession) findFeatureIDs(fs []database.Feature) ([]sql.NullInt64, err
f database.Feature
)
for rows.Next() {
err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat)
var typeID int
err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat, &typeID)
if err != nil {
return nil, handleError("querySearchFeatureID", err)
}

f.Type = types.byID[typeID]
fMap[f] = id
}

Expand Down

0 comments on commit 79af05e

Please sign in to comment.