From d3a1e55c294841d753f1de507e236aa931185b89 Mon Sep 17 00:00:00 2001 From: Sida Chen Date: Wed, 27 Feb 2019 18:02:46 -0500 Subject: [PATCH] database: Restructure database folder - one pgsql data model per folder, and one table per file - one data model per file --- database/ancestry.go | 82 ++++ database/feature.go | 82 ++++ database/layer.go | 45 ++ database/metadata.go | 27 ++ database/models.go | 345 --------------- database/namespace.go | 13 + database/notification.go | 55 +++ database/pgsql/ancestry.go | 361 ---------------- database/pgsql/ancestry/ancestry.go | 143 +++++++ database/pgsql/ancestry/ancestry_detector.go | 28 ++ database/pgsql/ancestry/ancestry_feature.go | 122 ++++++ database/pgsql/ancestry/ancestry_layer.go | 115 +++++ .../pgsql/{ => ancestry}/ancestry_test.go | 0 database/pgsql/{ => detector}/detector.go | 65 --- .../pgsql/{ => detector}/detector_test.go | 0 database/pgsql/entities/feature.go | 101 +++++ database/pgsql/{ => entities}/feature_test.go | 0 database/pgsql/{ => entities}/feature_type.go | 0 .../pgsql/{ => entities}/feature_type_test.go | 0 database/pgsql/{ => entities}/namespace.go | 0 .../pgsql/{ => entities}/namespace_test.go | 0 database/pgsql/entities/namespaced_feature.go | 131 ++++++ database/pgsql/feature.go | 398 ------------------ database/pgsql/{ => layer}/layer.go | 75 ---- database/pgsql/layer/layer_detector.go | 48 +++ database/pgsql/layer/layer_feature.go | 41 ++ database/pgsql/layer/layer_namespace.go | 40 ++ database/pgsql/{ => layer}/layer_test.go | 0 database/pgsql/{ => lock}/keyvalue.go | 0 database/pgsql/{ => lock}/keyvalue_test.go | 0 database/pgsql/{ => lock}/lock.go | 0 database/pgsql/{ => lock}/lock_test.go | 0 .../pgsql/{ => migrations}/migrations_test.go | 0 .../{ => notification}/notification_test.go | 0 .../vulnerability_notification.go} | 22 +- .../pgsql/{testdata => testutil}/data.sql | 0 database/pgsql/{ => testutil}/testutil.go | 0 database/pgsql/{ => util}/queries.go | 0 .../{ => vulnerability}/vulnerability.go | 35 -- .../vulnerability_affected_feature.go | 100 +++++ ...lnerability_affected_namespaced_feature.go | 116 +++++ .../{ => vulnerability}/vulnerability_test.go | 0 database/vulnerability.go | 36 ++ 43 files changed, 1336 insertions(+), 1290 deletions(-) create mode 100644 database/ancestry.go create mode 100644 database/feature.go create mode 100644 database/layer.go create mode 100644 database/metadata.go delete mode 100644 database/models.go create mode 100644 database/namespace.go create mode 100644 database/notification.go delete mode 100644 database/pgsql/ancestry.go create mode 100644 database/pgsql/ancestry/ancestry.go create mode 100644 database/pgsql/ancestry/ancestry_detector.go create mode 100644 database/pgsql/ancestry/ancestry_feature.go create mode 100644 database/pgsql/ancestry/ancestry_layer.go rename database/pgsql/{ => ancestry}/ancestry_test.go (100%) rename database/pgsql/{ => detector}/detector.go (62%) rename database/pgsql/{ => detector}/detector_test.go (100%) create mode 100644 database/pgsql/entities/feature.go rename database/pgsql/{ => entities}/feature_test.go (100%) rename database/pgsql/{ => entities}/feature_type.go (100%) rename database/pgsql/{ => entities}/feature_type_test.go (100%) rename database/pgsql/{ => entities}/namespace.go (100%) rename database/pgsql/{ => entities}/namespace_test.go (100%) create mode 100644 database/pgsql/entities/namespaced_feature.go delete mode 100644 database/pgsql/feature.go rename database/pgsql/{ => layer}/layer.go (78%) create mode 100644 database/pgsql/layer/layer_detector.go create mode 100644 database/pgsql/layer/layer_feature.go create mode 100644 database/pgsql/layer/layer_namespace.go rename database/pgsql/{ => layer}/layer_test.go (100%) rename database/pgsql/{ => lock}/keyvalue.go (100%) rename database/pgsql/{ => lock}/keyvalue_test.go (100%) rename database/pgsql/{ => lock}/lock.go (100%) rename database/pgsql/{ => lock}/lock_test.go (100%) rename database/pgsql/{ => migrations}/migrations_test.go (100%) rename database/pgsql/{ => notification}/notification_test.go (100%) rename database/pgsql/{notification.go => notification/vulnerability_notification.go} (96%) rename database/pgsql/{testdata => testutil}/data.sql (100%) rename database/pgsql/{ => testutil}/testutil.go (100%) rename database/pgsql/{ => util}/queries.go (100%) rename database/pgsql/{ => vulnerability}/vulnerability.go (90%) create mode 100644 database/pgsql/vulnerability/vulnerability_affected_feature.go create mode 100644 database/pgsql/vulnerability/vulnerability_affected_namespaced_feature.go rename database/pgsql/{ => vulnerability}/vulnerability_test.go (100%) create mode 100644 database/vulnerability.go diff --git a/database/ancestry.go b/database/ancestry.go new file mode 100644 index 0000000000..523c2bc85d --- /dev/null +++ b/database/ancestry.go @@ -0,0 +1,82 @@ +package database + +// Ancestry is a manifest that keeps all layers in an image in order. +type Ancestry struct { + // Name is a globally unique value for a set of layers. This is often the + // sha256 digest of an OCI/Docker manifest. + Name string `json:"name"` + // By contains the processors that are used when computing the + // content of this ancestry. + By []Detector `json:"by"` + // Layers should be ordered and i_th layer is the parent of i+1_th layer in + // the slice. + Layers []AncestryLayer `json:"layers"` +} + +// Valid checks if the ancestry is compliant to spec. +func (a *Ancestry) Valid() bool { + if a == nil { + return false + } + + if a.Name == "" { + return false + } + + for _, d := range a.By { + if !d.Valid() { + return false + } + } + + for _, l := range a.Layers { + if !l.Valid() { + return false + } + } + + return true +} + +// AncestryLayer is a layer with all detected namespaced features. +type AncestryLayer struct { + // Hash is the sha-256 tarsum on the layer's blob content. + Hash string `json:"hash"` + // Features are the features introduced by this layer when it was + // processed. + Features []AncestryFeature `json:"features"` +} + +// Valid checks if the Ancestry Layer is compliant to the spec. +func (l *AncestryLayer) Valid() bool { + if l == nil { + return false + } + + if l.Hash == "" { + return false + } + + return true +} + +// GetFeatures returns the Ancestry's features. +func (l *AncestryLayer) GetFeatures() []NamespacedFeature { + nsf := make([]NamespacedFeature, 0, len(l.Features)) + for _, f := range l.Features { + nsf = append(nsf, f.NamespacedFeature) + } + + return nsf +} + +// AncestryFeature is a namespaced feature with the detectors used to +// find this feature. +type AncestryFeature struct { + NamespacedFeature `json:"namespacedFeature"` + + // FeatureBy is the detector that detected the feature. + FeatureBy Detector `json:"featureBy"` + // NamespaceBy is the detector that detected the namespace. + NamespaceBy Detector `json:"namespaceBy"` +} diff --git a/database/feature.go b/database/feature.go new file mode 100644 index 0000000000..8bd408885c --- /dev/null +++ b/database/feature.go @@ -0,0 +1,82 @@ +package database + +// Feature represents a package detected in a layer but the namespace is not +// determined. +// +// e.g. Name: Libssl1.0, Version: 1.0, VersionFormat: dpkg, Type: binary +// dpkg is the version format of the installer package manager, which in this +// case could be dpkg or apk. +type Feature struct { + Name string `json:"name"` + Version string `json:"version"` + VersionFormat string `json:"versionFormat"` + Type FeatureType `json:"type"` +} + +// NamespacedFeature is a feature with determined namespace and can be affected +// by vulnerabilities. +// +// e.g. OpenSSL 1.0 dpkg Debian:7. +type NamespacedFeature struct { + Feature `json:"feature"` + + Namespace Namespace `json:"namespace"` +} + +// AffectedNamespacedFeature is a namespaced feature affected by the +// vulnerabilities with fixed-in versions for this feature. +type AffectedNamespacedFeature struct { + NamespacedFeature + + AffectedBy []VulnerabilityWithFixedIn +} + +// VulnerabilityWithFixedIn is used for AffectedNamespacedFeature to retrieve +// the affecting vulnerabilities and the fixed-in versions for the feature. +type VulnerabilityWithFixedIn struct { + Vulnerability + + FixedInVersion string +} + +// AffectedFeature is used to determine whether a namespaced feature is affected +// by a Vulnerability. Namespace and Feature Name is unique. Affected Feature is +// bound to vulnerability. +type AffectedFeature struct { + // FeatureType determines which type of package it affects. + FeatureType FeatureType + Namespace Namespace + FeatureName string + // FixedInVersion is known next feature version that's not affected by the + // vulnerability. Empty FixedInVersion means the unaffected version is + // unknown. + FixedInVersion string + // AffectedVersion contains the version range to determine whether or not a + // feature is affected. + AffectedVersion string +} + +// NullableAffectedNamespacedFeature is an affectednamespacedfeature with +// whether it's found in datastore. +type NullableAffectedNamespacedFeature struct { + AffectedNamespacedFeature + + Valid bool +} + +func NewFeature(name string, version string, versionFormat string, featureType FeatureType) *Feature { + return &Feature{name, version, versionFormat, featureType} +} + +func NewBinaryPackage(name string, version string, versionFormat string) *Feature { + return &Feature{name, version, versionFormat, BinaryPackage} +} + +func NewSourcePackage(name string, version string, versionFormat string) *Feature { + return &Feature{name, version, versionFormat, SourcePackage} +} + +func NewNamespacedFeature(namespace *Namespace, feature *Feature) *NamespacedFeature { + // TODO: namespaced feature should use pointer values + return &NamespacedFeature{*feature, *namespace} +} diff --git a/database/layer.go b/database/layer.go new file mode 100644 index 0000000000..c68dac4c54 --- /dev/null +++ b/database/layer.go @@ -0,0 +1,45 @@ +package database + +// Layer is a layer with all the detected features and namespaces. +type Layer struct { + // Hash is the sha-256 tarsum on the layer's blob content. + Hash string `json:"hash"` + // By contains a list of detectors scanned this Layer. + By []Detector `json:"by"` + Namespaces []LayerNamespace `json:"namespaces"` + Features []LayerFeature `json:"features"` +} + +func (l *Layer) GetFeatures() []Feature { + features := make([]Feature, 0, len(l.Features)) + for _, f := range l.Features { + features = append(features, f.Feature) + } + + return features +} + +func (l *Layer) GetNamespaces() []Namespace { + namespaces := make([]Namespace, 0, len(l.Namespaces)) + for _, ns := range l.Namespaces { + namespaces = append(namespaces, ns.Namespace) + } + + return namespaces +} + +// LayerNamespace is a namespace with detection information. +type LayerNamespace struct { + Namespace `json:"namespace"` + + // By is the detector found the namespace. + By Detector `json:"by"` +} + +// LayerFeature is a feature with detection information. +type LayerFeature struct { + Feature `json:"feature"` + + // By is the detector found the feature. + By Detector `json:"by"` +} diff --git a/database/metadata.go b/database/metadata.go new file mode 100644 index 0000000000..7167c4a173 --- /dev/null +++ b/database/metadata.go @@ -0,0 +1,27 @@ +package database + +import ( + "database/sql/driver" + "encoding/json" +) + +// MetadataMap is for storing the metadata returned by vulnerability database. +type MetadataMap map[string]interface{} + +func (mm *MetadataMap) Scan(value interface{}) error { + if value == nil { + return nil + } + + // github.com/lib/pq decodes TEXT/VARCHAR fields into strings. + val, ok := value.(string) + if !ok { + panic("got type other than []byte from database") + } + return json.Unmarshal([]byte(val), mm) +} + +func (mm *MetadataMap) Value() (driver.Value, error) { + json, err := json.Marshal(*mm) + return string(json), err +} diff --git a/database/models.go b/database/models.go deleted file mode 100644 index 9abba3c0d8..0000000000 --- a/database/models.go +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright 2017 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package database - -import ( - "database/sql/driver" - "encoding/json" - "time" - - "github.com/coreos/clair/pkg/pagination" -) - -// Ancestry is a manifest that keeps all layers in an image in order. -type Ancestry struct { - // Name is a globally unique value for a set of layers. This is often the - // sha256 digest of an OCI/Docker manifest. - Name string `json:"name"` - // By contains the processors that are used when computing the - // content of this ancestry. - By []Detector `json:"by"` - // Layers should be ordered and i_th layer is the parent of i+1_th layer in - // the slice. - Layers []AncestryLayer `json:"layers"` -} - -// Valid checks if the ancestry is compliant to spec. -func (a *Ancestry) Valid() bool { - if a == nil { - return false - } - - if a.Name == "" { - return false - } - - for _, d := range a.By { - if !d.Valid() { - return false - } - } - - for _, l := range a.Layers { - if !l.Valid() { - return false - } - } - - return true -} - -// AncestryLayer is a layer with all detected namespaced features. -type AncestryLayer struct { - // Hash is the sha-256 tarsum on the layer's blob content. - Hash string `json:"hash"` - // Features are the features introduced by this layer when it was - // processed. - Features []AncestryFeature `json:"features"` -} - -// Valid checks if the Ancestry Layer is compliant to the spec. -func (l *AncestryLayer) Valid() bool { - if l == nil { - return false - } - - if l.Hash == "" { - return false - } - - return true -} - -// GetFeatures returns the Ancestry's features. -func (l *AncestryLayer) GetFeatures() []NamespacedFeature { - nsf := make([]NamespacedFeature, 0, len(l.Features)) - for _, f := range l.Features { - nsf = append(nsf, f.NamespacedFeature) - } - - return nsf -} - -// AncestryFeature is a namespaced feature with the detectors used to -// find this feature. -type AncestryFeature struct { - NamespacedFeature `json:"namespacedFeature"` - - // FeatureBy is the detector that detected the feature. - FeatureBy Detector `json:"featureBy"` - // NamespaceBy is the detector that detected the namespace. - NamespaceBy Detector `json:"namespaceBy"` -} - -// Layer is a layer with all the detected features and namespaces. -type Layer struct { - // Hash is the sha-256 tarsum on the layer's blob content. - Hash string `json:"hash"` - // By contains a list of detectors scanned this Layer. - By []Detector `json:"by"` - Namespaces []LayerNamespace `json:"namespaces"` - Features []LayerFeature `json:"features"` -} - -func (l *Layer) GetFeatures() []Feature { - features := make([]Feature, 0, len(l.Features)) - for _, f := range l.Features { - features = append(features, f.Feature) - } - - return features -} - -func (l *Layer) GetNamespaces() []Namespace { - namespaces := make([]Namespace, 0, len(l.Namespaces)) - for _, ns := range l.Namespaces { - namespaces = append(namespaces, ns.Namespace) - } - - return namespaces -} - -// LayerNamespace is a namespace with detection information. -type LayerNamespace struct { - Namespace `json:"namespace"` - - // By is the detector found the namespace. - By Detector `json:"by"` -} - -// LayerFeature is a feature with detection information. -type LayerFeature struct { - Feature `json:"feature"` - - // By is the detector found the feature. - By Detector `json:"by"` -} - -// Namespace is the contextual information around features. -// -// e.g. Debian:7, NodeJS. -type Namespace struct { - Name string `json:"name"` - VersionFormat string `json:"versionFormat"` -} - -func NewNamespace(name string, versionFormat string) *Namespace { - return &Namespace{name, versionFormat} -} - -// Feature represents a package detected in a layer but the namespace is not -// determined. -// -// e.g. Name: Libssl1.0, Version: 1.0, VersionFormat: dpkg, Type: binary -// dpkg is the version format of the installer package manager, which in this -// case could be dpkg or apk. -type Feature struct { - Name string `json:"name"` - Version string `json:"version"` - VersionFormat string `json:"versionFormat"` - Type FeatureType `json:"type"` -} - -func NewFeature(name string, version string, versionFormat string, featureType FeatureType) *Feature { - return &Feature{name, version, versionFormat, featureType} -} - -func NewBinaryPackage(name string, version string, versionFormat string) *Feature { - return &Feature{name, version, versionFormat, BinaryPackage} -} - -func NewSourcePackage(name string, version string, versionFormat string) *Feature { - return &Feature{name, version, versionFormat, SourcePackage} -} - -// NamespacedFeature is a feature with determined namespace and can be affected -// by vulnerabilities. -// -// e.g. OpenSSL 1.0 dpkg Debian:7. -type NamespacedFeature struct { - Feature `json:"feature"` - - Namespace Namespace `json:"namespace"` -} - -func NewNamespacedFeature(namespace *Namespace, feature *Feature) *NamespacedFeature { - // TODO: namespaced feature should use pointer values - return &NamespacedFeature{*feature, *namespace} -} - -// AffectedNamespacedFeature is a namespaced feature affected by the -// vulnerabilities with fixed-in versions for this feature. -type AffectedNamespacedFeature struct { - NamespacedFeature - - AffectedBy []VulnerabilityWithFixedIn -} - -// VulnerabilityWithFixedIn is used for AffectedNamespacedFeature to retrieve -// the affecting vulnerabilities and the fixed-in versions for the feature. -type VulnerabilityWithFixedIn struct { - Vulnerability - - FixedInVersion string -} - -// AffectedFeature is used to determine whether a namespaced feature is affected -// by a Vulnerability. Namespace and Feature Name is unique. Affected Feature is -// bound to vulnerability. -type AffectedFeature struct { - // FeatureType determines which type of package it affects. - FeatureType FeatureType - Namespace Namespace - FeatureName string - // FixedInVersion is known next feature version that's not affected by the - // vulnerability. Empty FixedInVersion means the unaffected version is - // unknown. - FixedInVersion string - // AffectedVersion contains the version range to determine whether or not a - // feature is affected. - AffectedVersion string -} - -// VulnerabilityID is an identifier for every vulnerability. Every vulnerability -// has unique namespace and name. -type VulnerabilityID struct { - Name string - Namespace string -} - -// Vulnerability represents CVE or similar vulnerability reports. -type Vulnerability struct { - Name string - Namespace Namespace - - Description string - Link string - Severity Severity - - Metadata MetadataMap -} - -// VulnerabilityWithAffected is a vulnerability with all known affected -// features. -type VulnerabilityWithAffected struct { - Vulnerability - - Affected []AffectedFeature -} - -// PagedVulnerableAncestries is a vulnerability with a page of affected -// ancestries each with a special index attached for streaming purpose. The -// current page number and next page number are for navigate. -type PagedVulnerableAncestries struct { - Vulnerability - - // Affected is a map of special indexes to Ancestries, which the pair - // should be unique in a stream. Every indexes in the map should be larger - // than previous page. - Affected map[int]string - - Limit int - Current pagination.Token - Next pagination.Token - - // End signals the end of the pages. - End bool -} - -// NotificationHook is a message sent to another service to inform of a change -// to a Vulnerability or the Ancestries affected by a Vulnerability. It contains -// the name of a notification that should be read and marked as read via the -// API. -type NotificationHook struct { - Name string - - Created time.Time - Notified time.Time - Deleted time.Time -} - -// VulnerabilityNotification is a notification for vulnerability changes. -type VulnerabilityNotification struct { - NotificationHook - - Old *Vulnerability - New *Vulnerability -} - -// VulnerabilityNotificationWithVulnerable is a notification for vulnerability -// changes with vulnerable ancestries. -type VulnerabilityNotificationWithVulnerable struct { - NotificationHook - - Old *PagedVulnerableAncestries - New *PagedVulnerableAncestries -} - -// MetadataMap is for storing the metadata returned by vulnerability database. -type MetadataMap map[string]interface{} - -// NullableAffectedNamespacedFeature is an affectednamespacedfeature with -// whether it's found in datastore. -type NullableAffectedNamespacedFeature struct { - AffectedNamespacedFeature - - Valid bool -} - -// NullableVulnerability is a vulnerability with whether the vulnerability is -// found in datastore. -type NullableVulnerability struct { - VulnerabilityWithAffected - - Valid bool -} - -func (mm *MetadataMap) Scan(value interface{}) error { - if value == nil { - return nil - } - - // github.com/lib/pq decodes TEXT/VARCHAR fields into strings. - val, ok := value.(string) - if !ok { - panic("got type other than []byte from database") - } - return json.Unmarshal([]byte(val), mm) -} - -func (mm *MetadataMap) Value() (driver.Value, error) { - json, err := json.Marshal(*mm) - return string(json), err -} diff --git a/database/namespace.go b/database/namespace.go new file mode 100644 index 0000000000..6d384c1399 --- /dev/null +++ b/database/namespace.go @@ -0,0 +1,13 @@ +package database + +// Namespace is the contextual information around features. +// +// e.g. Debian:7, NodeJS. +type Namespace struct { + Name string `json:"name"` + VersionFormat string `json:"versionFormat"` +} + +func NewNamespace(name string, versionFormat string) *Namespace { + return &Namespace{name, versionFormat} +} diff --git a/database/notification.go b/database/notification.go new file mode 100644 index 0000000000..9443cd7196 --- /dev/null +++ b/database/notification.go @@ -0,0 +1,55 @@ +package database + +import ( + "time" + + "github.com/coreos/clair/pkg/pagination" +) + +// NotificationHook is a message sent to another service to inform of a change +// to a Vulnerability or the Ancestries affected by a Vulnerability. It contains +// the name of a notification that should be read and marked as read via the +// API. +type NotificationHook struct { + Name string + + Created time.Time + Notified time.Time + Deleted time.Time +} + +// VulnerabilityNotification is a notification for vulnerability changes. +type VulnerabilityNotification struct { + NotificationHook + + Old *Vulnerability + New *Vulnerability +} + +// VulnerabilityNotificationWithVulnerable is a notification for vulnerability +// changes with vulnerable ancestries. +type VulnerabilityNotificationWithVulnerable struct { + NotificationHook + + Old *PagedVulnerableAncestries + New *PagedVulnerableAncestries +} + +// PagedVulnerableAncestries is a vulnerability with a page of affected +// ancestries each with a special index attached for streaming purpose. The +// current page number and next page number are for navigate. +type PagedVulnerableAncestries struct { + Vulnerability + + // Affected is a map of special indexes to Ancestries, which the pair + // should be unique in a stream. Every indexes in the map should be larger + // than previous page. + Affected map[int]string + + Limit int + Current pagination.Token + Next pagination.Token + + // End signals the end of the pages. + End bool +} diff --git a/database/pgsql/ancestry.go b/database/pgsql/ancestry.go deleted file mode 100644 index 992e9dd31e..0000000000 --- a/database/pgsql/ancestry.go +++ /dev/null @@ -1,361 +0,0 @@ -package pgsql - -import ( - "database/sql" - "errors" - - log "github.com/sirupsen/logrus" - - "github.com/coreos/clair/database" - "github.com/coreos/clair/pkg/commonerr" -) - -const ( - insertAncestry = ` - INSERT INTO ancestry (name) VALUES ($1) RETURNING id` - - findAncestryLayerHashes = ` - SELECT layer.hash, ancestry_layer.ancestry_index - FROM layer, ancestry_layer - WHERE ancestry_layer.ancestry_id = $1 - AND ancestry_layer.layer_id = layer.id - ORDER BY ancestry_layer.ancestry_index ASC` - - findAncestryFeatures = ` - SELECT namespace.name, namespace.version_format, feature.name, - feature.version, feature.version_format, feature_type.name, ancestry_layer.ancestry_index, - ancestry_feature.feature_detector_id, ancestry_feature.namespace_detector_id - FROM namespace, feature, feature_type, namespaced_feature, ancestry_layer, ancestry_feature - WHERE ancestry_layer.ancestry_id = $1 - AND feature_type.id = feature.type - AND ancestry_feature.ancestry_layer_id = ancestry_layer.id - AND ancestry_feature.namespaced_feature_id = namespaced_feature.id - AND namespaced_feature.feature_id = feature.id - AND namespaced_feature.namespace_id = namespace.id` - - findAncestryID = `SELECT id FROM ancestry WHERE name = $1` - removeAncestry = `DELETE FROM ancestry WHERE name = $1` - insertAncestryLayers = ` - INSERT INTO ancestry_layer (ancestry_id, ancestry_index, layer_id) VALUES ($1, $2, $3) - RETURNING id` - insertAncestryFeatures = ` - INSERT INTO ancestry_feature - (ancestry_layer_id, namespaced_feature_id, feature_detector_id, namespace_detector_id) VALUES - ($1, $2, $3, $4)` -) - -func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { - var ( - ancestry = database.Ancestry{Name: name} - err error - ) - - id, ok, err := tx.findAncestryID(name) - if !ok || err != nil { - return ancestry, ok, err - } - - if ancestry.By, err = tx.findAncestryDetectors(id); err != nil { - return ancestry, false, err - } - - if ancestry.Layers, err = tx.findAncestryLayers(id); err != nil { - return ancestry, false, err - } - - return ancestry, true, nil -} - -func (tx *pgSession) UpsertAncestry(ancestry database.Ancestry) error { - if !ancestry.Valid() { - return database.ErrInvalidParameters - } - - if err := tx.removeAncestry(ancestry.Name); err != nil { - return err - } - - id, err := tx.insertAncestry(ancestry.Name) - if err != nil { - return err - } - - detectorIDs, err := tx.findDetectorIDs(ancestry.By) - if err != nil { - return err - } - - // insert ancestry metadata - if err := tx.insertAncestryDetectors(id, detectorIDs); err != nil { - return err - } - - layers := make([]string, 0, len(ancestry.Layers)) - for _, layer := range ancestry.Layers { - layers = append(layers, layer.Hash) - } - - layerIDs, ok, err := tx.findLayerIDs(layers) - if err != nil { - return err - } - - if !ok { - log.Error("layer cannot be found, this indicates that the internal logic of calling UpsertAncestry is wrong or the database is corrupted.") - return database.ErrMissingEntities - } - - ancestryLayerIDs, err := tx.insertAncestryLayers(id, layerIDs) - if err != nil { - return err - } - - for i, id := range ancestryLayerIDs { - if err := tx.insertAncestryFeatures(id, ancestry.Layers[i]); err != nil { - return err - } - } - - return nil -} - -func (tx *pgSession) insertAncestry(name string) (int64, error) { - var id int64 - err := tx.QueryRow(insertAncestry, name).Scan(&id) - if err != nil { - if isErrUniqueViolation(err) { - return 0, handleError("insertAncestry", errors.New("other Go-routine is processing this ancestry (skip)")) - } - - return 0, handleError("insertAncestry", err) - } - - return id, nil -} - -func (tx *pgSession) findAncestryID(name string) (int64, bool, error) { - var id sql.NullInt64 - if err := tx.QueryRow(findAncestryID, name).Scan(&id); err != nil { - if err == sql.ErrNoRows { - return 0, false, nil - } - - return 0, false, handleError("findAncestryID", err) - } - - return id.Int64, true, nil -} - -func (tx *pgSession) removeAncestry(name string) error { - result, err := tx.Exec(removeAncestry, name) - if err != nil { - return handleError("removeAncestry", err) - } - - affected, err := result.RowsAffected() - if err != nil { - return handleError("removeAncestry", err) - } - - if affected != 0 { - log.WithField("ancestry", name).Debug("removed ancestry") - } - - return nil -} - -func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, error) { - detectors, err := tx.findAllDetectors() - if err != nil { - return nil, err - } - - layerMap, err := tx.findAncestryLayerHashes(id) - if err != nil { - return nil, err - } - - featureMap, err := tx.findAncestryFeatures(id, detectors) - if err != nil { - return nil, err - } - - layers := make([]database.AncestryLayer, len(layerMap)) - for index, layer := range layerMap { - // index MUST match the ancestry layer slice index. - if layers[index].Hash == "" && len(layers[index].Features) == 0 { - layers[index] = database.AncestryLayer{ - Hash: layer, - Features: featureMap[index], - } - } else { - log.WithFields(log.Fields{ - "ancestry ID": id, - "duplicated ancestry index": index, - }).WithError(database.ErrInconsistent).Error("ancestry layers with same ancestry_index is not allowed") - return nil, database.ErrInconsistent - } - } - - return layers, nil -} - -func (tx *pgSession) findAncestryLayerHashes(ancestryID int64) (map[int64]string, error) { - // retrieve layer indexes and hashes - rows, err := tx.Query(findAncestryLayerHashes, ancestryID) - if err != nil { - return nil, handleError("findAncestryLayerHashes", err) - } - - layerHashes := map[int64]string{} - for rows.Next() { - var ( - hash string - index int64 - ) - - if err = rows.Scan(&hash, &index); err != nil { - return nil, handleError("findAncestryLayerHashes", err) - } - - if _, ok := layerHashes[index]; ok { - // one ancestry index should correspond to only one layer - return nil, database.ErrInconsistent - } - - layerHashes[index] = hash - } - - return layerHashes, nil -} - -func (tx *pgSession) findAncestryFeatures(ancestryID int64, detectors detectorMap) (map[int64][]database.AncestryFeature, error) { - // ancestry_index -> ancestry features - featureMap := make(map[int64][]database.AncestryFeature) - // retrieve ancestry layer's namespaced features - rows, err := tx.Query(findAncestryFeatures, ancestryID) - if err != nil { - return nil, handleError("findAncestryFeatures", err) - } - - defer rows.Close() - - for rows.Next() { - var ( - featureDetectorID int64 - namespaceDetectorID int64 - feature database.NamespacedFeature - // index is used to determine which layer the feature belongs to. - index sql.NullInt64 - ) - - if err := rows.Scan( - &feature.Namespace.Name, - &feature.Namespace.VersionFormat, - &feature.Feature.Name, - &feature.Feature.Version, - &feature.Feature.VersionFormat, - &feature.Feature.Type, - &index, - &featureDetectorID, - &namespaceDetectorID, - ); err != nil { - return nil, handleError("findAncestryFeatures", err) - } - - if feature.Feature.VersionFormat != feature.Namespace.VersionFormat { - // Feature must have the same version format as the associated - // namespace version format. - return nil, database.ErrInconsistent - } - - fDetector, ok := detectors.byID[featureDetectorID] - if !ok { - return nil, database.ErrInconsistent - } - - nsDetector, ok := detectors.byID[namespaceDetectorID] - if !ok { - return nil, database.ErrInconsistent - } - - featureMap[index.Int64] = append(featureMap[index.Int64], database.AncestryFeature{ - NamespacedFeature: feature, - FeatureBy: fDetector, - NamespaceBy: nsDetector, - }) - } - - return featureMap, nil -} - -// insertAncestryLayers inserts the ancestry layers along with its content into -// the database. The layers are 0 based indexed in the original order. -func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []int64) ([]int64, error) { - stmt, err := tx.Prepare(insertAncestryLayers) - if err != nil { - return nil, handleError("insertAncestryLayers", err) - } - - ancestryLayerIDs := []int64{} - for index, layerID := range layers { - var ancestryLayerID sql.NullInt64 - if err := stmt.QueryRow(ancestryID, index, layerID).Scan(&ancestryLayerID); err != nil { - return nil, handleError("insertAncestryLayers", commonerr.CombineErrors(err, stmt.Close())) - } - - if !ancestryLayerID.Valid { - return nil, database.ErrInconsistent - } - - ancestryLayerIDs = append(ancestryLayerIDs, ancestryLayerID.Int64) - } - - if err := stmt.Close(); err != nil { - return nil, handleError("insertAncestryLayers", err) - } - - return ancestryLayerIDs, nil -} - -func (tx *pgSession) insertAncestryFeatures(ancestryLayerID int64, layer database.AncestryLayer) error { - detectors, err := tx.findAllDetectors() - if err != nil { - return err - } - - nsFeatureIDs, err := tx.findNamespacedFeatureIDs(layer.GetFeatures()) - if err != nil { - return err - } - - // find the detectors for each feature - stmt, err := tx.Prepare(insertAncestryFeatures) - if err != nil { - return handleError("insertAncestryFeatures", err) - } - - defer stmt.Close() - - for index, id := range nsFeatureIDs { - if !id.Valid { - return database.ErrMissingEntities - } - - namespaceDetectorID, ok := detectors.byValue[layer.Features[index].NamespaceBy] - if !ok { - return database.ErrMissingEntities - } - - featureDetectorID, ok := detectors.byValue[layer.Features[index].FeatureBy] - if !ok { - return database.ErrMissingEntities - } - - if _, err := stmt.Exec(ancestryLayerID, id, featureDetectorID, namespaceDetectorID); err != nil { - return handleError("insertAncestryFeatures", commonerr.CombineErrors(err, stmt.Close())) - } - } - - return nil -} diff --git a/database/pgsql/ancestry/ancestry.go b/database/pgsql/ancestry/ancestry.go new file mode 100644 index 0000000000..feeb56af49 --- /dev/null +++ b/database/pgsql/ancestry/ancestry.go @@ -0,0 +1,143 @@ +package pgsql + +import ( + "database/sql" + "errors" + + log "github.com/sirupsen/logrus" + + "github.com/coreos/clair/database" +) + +const ( + insertAncestry = ` + INSERT INTO ancestry (name) VALUES ($1) RETURNING id` + + findAncestryID = `SELECT id FROM ancestry WHERE name = $1` + removeAncestry = `DELETE FROM ancestry WHERE name = $1` + + insertAncestryFeatures = ` + INSERT INTO ancestry_feature + (ancestry_layer_id, namespaced_feature_id, feature_detector_id, namespace_detector_id) VALUES + ($1, $2, $3, $4)` +) + +func (tx *pgSession) FindAncestry(name string) (database.Ancestry, bool, error) { + var ( + ancestry = database.Ancestry{Name: name} + err error + ) + + id, ok, err := tx.findAncestryID(name) + if !ok || err != nil { + return ancestry, ok, err + } + + if ancestry.By, err = tx.findAncestryDetectors(id); err != nil { + return ancestry, false, err + } + + if ancestry.Layers, err = tx.findAncestryLayers(id); err != nil { + return ancestry, false, err + } + + return ancestry, true, nil +} + +func (tx *pgSession) UpsertAncestry(ancestry database.Ancestry) error { + if !ancestry.Valid() { + return database.ErrInvalidParameters + } + + if err := tx.removeAncestry(ancestry.Name); err != nil { + return err + } + + id, err := tx.insertAncestry(ancestry.Name) + if err != nil { + return err + } + + detectorIDs, err := tx.findDetectorIDs(ancestry.By) + if err != nil { + return err + } + + // insert ancestry metadata + if err := tx.insertAncestryDetectors(id, detectorIDs); err != nil { + return err + } + + layers := make([]string, 0, len(ancestry.Layers)) + for _, layer := range ancestry.Layers { + layers = append(layers, layer.Hash) + } + + layerIDs, ok, err := tx.findLayerIDs(layers) + if err != nil { + return err + } + + if !ok { + log.Error("layer cannot be found, this indicates that the internal logic of calling UpsertAncestry is wrong or the database is corrupted.") + return database.ErrMissingEntities + } + + ancestryLayerIDs, err := tx.insertAncestryLayers(id, layerIDs) + if err != nil { + return err + } + + for i, id := range ancestryLayerIDs { + if err := tx.insertAncestryFeatures(id, ancestry.Layers[i]); err != nil { + return err + } + } + + return nil +} + +func (tx *pgSession) insertAncestry(name string) (int64, error) { + var id int64 + err := tx.QueryRow(insertAncestry, name).Scan(&id) + if err != nil { + if isErrUniqueViolation(err) { + return 0, handleError("insertAncestry", errors.New("other Go-routine is processing this ancestry (skip)")) + } + + return 0, handleError("insertAncestry", err) + } + + return id, nil +} + +func (tx *pgSession) findAncestryID(name string) (int64, bool, error) { + var id sql.NullInt64 + if err := tx.QueryRow(findAncestryID, name).Scan(&id); err != nil { + if err == sql.ErrNoRows { + return 0, false, nil + } + + return 0, false, handleError("findAncestryID", err) + } + + return id.Int64, true, nil +} + +func (tx *pgSession) removeAncestry(name string) error { + result, err := tx.Exec(removeAncestry, name) + if err != nil { + return handleError("removeAncestry", err) + } + + affected, err := result.RowsAffected() + if err != nil { + return handleError("removeAncestry", err) + } + + if affected != 0 { + log.WithField("ancestry", name).Debug("removed ancestry") + } + + return nil +} diff --git a/database/pgsql/ancestry/ancestry_detector.go b/database/pgsql/ancestry/ancestry_detector.go new file mode 100644 index 0000000000..2becaae51e --- /dev/null +++ b/database/pgsql/ancestry/ancestry_detector.go @@ -0,0 +1,28 @@ +package pgsql + +import "github.com/coreos/clair/database" + +var selectAncestryDetectors = ` +SELECT d.name, d.version, d.dtype + FROM ancestry_detector, detector AS d + WHERE ancestry_detector.detector_id = d.id AND ancestry_detector.ancestry_id = $1;` + +var insertAncestryDetectors = ` + INSERT INTO ancestry_detector (ancestry_id, detector_id) + SELECT $1, $2 + WHERE NOT EXISTS (SELECT id FROM ancestry_detector WHERE ancestry_id = $1 AND detector_id = $2)` + +func (tx *pgSession) findAncestryDetectors(id int64) ([]database.Detector, error) { + detectors, err := tx.getDetectors(selectAncestryDetectors, id) + return detectors, err +} + +func (tx *pgSession) insertAncestryDetectors(ancestryID int64, detectorIDs []int64) error { + for _, detectorID := range detectorIDs { + if _, err := tx.Exec(insertAncestryDetectors, ancestryID, detectorID); err != nil { + return handleError("insertAncestryDetectors", err) + } + } + + return nil +} diff --git a/database/pgsql/ancestry/ancestry_feature.go b/database/pgsql/ancestry/ancestry_feature.go new file mode 100644 index 0000000000..663e9494a1 --- /dev/null +++ b/database/pgsql/ancestry/ancestry_feature.go @@ -0,0 +1,122 @@ +package pgsql + +import ( + "database/sql" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/commonerr" +) + +const findAncestryFeatures = ` + SELECT namespace.name, namespace.version_format, feature.name, + feature.version, feature.version_format, feature_type.name, ancestry_layer.ancestry_index, + ancestry_feature.feature_detector_id, ancestry_feature.namespace_detector_id + FROM namespace, feature, feature_type, namespaced_feature, ancestry_layer, ancestry_feature + WHERE ancestry_layer.ancestry_id = $1 + AND feature_type.id = feature.type + AND ancestry_feature.ancestry_layer_id = ancestry_layer.id + AND ancestry_feature.namespaced_feature_id = namespaced_feature.id + AND namespaced_feature.feature_id = feature.id + AND namespaced_feature.namespace_id = namespace.id` + +func (tx *pgSession) findAncestryFeatures(ancestryID int64, detectors detectorMap) (map[int64][]database.AncestryFeature, error) { + // ancestry_index -> ancestry features + featureMap := make(map[int64][]database.AncestryFeature) + // retrieve ancestry layer's namespaced features + rows, err := tx.Query(findAncestryFeatures, ancestryID) + if err != nil { + return nil, handleError("findAncestryFeatures", err) + } + + defer rows.Close() + + for rows.Next() { + var ( + featureDetectorID int64 + namespaceDetectorID int64 + feature database.NamespacedFeature + // index is used to determine which layer the feature belongs to. + index sql.NullInt64 + ) + + if err := rows.Scan( + &feature.Namespace.Name, + &feature.Namespace.VersionFormat, + &feature.Feature.Name, + &feature.Feature.Version, + &feature.Feature.VersionFormat, + &feature.Feature.Type, + &index, + &featureDetectorID, + &namespaceDetectorID, + ); err != nil { + return nil, handleError("findAncestryFeatures", err) + } + + if feature.Feature.VersionFormat != feature.Namespace.VersionFormat { + // Feature must have the same version format as the associated + // namespace version format. + return nil, database.ErrInconsistent + } + + fDetector, ok := detectors.byID[featureDetectorID] + if !ok { + return nil, database.ErrInconsistent + } + + nsDetector, ok := detectors.byID[namespaceDetectorID] + if !ok { + return nil, database.ErrInconsistent + } + + featureMap[index.Int64] = append(featureMap[index.Int64], database.AncestryFeature{ + NamespacedFeature: feature, + FeatureBy: fDetector, + NamespaceBy: nsDetector, + }) + } + + return featureMap, nil +} + +func (tx *pgSession) insertAncestryFeatures(ancestryLayerID int64, layer database.AncestryLayer) error { + detectors, err := tx.findAllDetectors() + if err != nil { + return err + } + + nsFeatureIDs, err := tx.findNamespacedFeatureIDs(layer.GetFeatures()) + if err != nil { + return err + } + + // find the detectors for each feature + stmt, err := tx.Prepare(insertAncestryFeatures) + if err != nil { + return handleError("insertAncestryFeatures", err) + } + + defer stmt.Close() + + for index, id := range nsFeatureIDs { + if !id.Valid { + return database.ErrMissingEntities + } + + namespaceDetectorID, ok := detectors.byValue[layer.Features[index].NamespaceBy] + if !ok { + return database.ErrMissingEntities + } + + featureDetectorID, ok := detectors.byValue[layer.Features[index].FeatureBy] + if !ok { + return database.ErrMissingEntities + } + + if _, err := stmt.Exec(ancestryLayerID, id, featureDetectorID, namespaceDetectorID); err != nil { + return handleError("insertAncestryFeatures", commonerr.CombineErrors(err, stmt.Close())) + } + } + + return nil +} diff --git a/database/pgsql/ancestry/ancestry_layer.go b/database/pgsql/ancestry/ancestry_layer.go new file mode 100644 index 0000000000..09505d0167 --- /dev/null +++ b/database/pgsql/ancestry/ancestry_layer.go @@ -0,0 +1,115 @@ +package pgsql + +import ( + "database/sql" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/commonerr" + log "github.com/sirupsen/logrus" +) + +const ( + findAncestryLayerHashes = ` + SELECT layer.hash, ancestry_layer.ancestry_index + FROM layer, ancestry_layer + WHERE ancestry_layer.ancestry_id = $1 + AND ancestry_layer.layer_id = layer.id + ORDER BY ancestry_layer.ancestry_index ASC` + insertAncestryLayers = ` + INSERT INTO ancestry_layer (ancestry_id, ancestry_index, layer_id) VALUES ($1, $2, $3) + RETURNING id` +) + +func (tx *pgSession) findAncestryLayerHashes(ancestryID int64) (map[int64]string, error) { + // retrieve layer indexes and hashes + rows, err := tx.Query(findAncestryLayerHashes, ancestryID) + if err != nil { + return nil, handleError("findAncestryLayerHashes", err) + } + + layerHashes := map[int64]string{} + for rows.Next() { + var ( + hash string + index int64 + ) + + if err = rows.Scan(&hash, &index); err != nil { + return nil, handleError("findAncestryLayerHashes", err) + } + + if _, ok := layerHashes[index]; ok { + // one ancestry index should correspond to only one layer + return nil, database.ErrInconsistent + } + + layerHashes[index] = hash + } + + return layerHashes, nil +} + +// insertAncestryLayers inserts the ancestry layers along with its content into +// the database. The layers are 0 based indexed in the original order. +func (tx *pgSession) insertAncestryLayers(ancestryID int64, layers []int64) ([]int64, error) { + stmt, err := tx.Prepare(insertAncestryLayers) + if err != nil { + return nil, handleError("insertAncestryLayers", err) + } + + ancestryLayerIDs := []int64{} + for index, layerID := range layers { + var ancestryLayerID sql.NullInt64 + if err := stmt.QueryRow(ancestryID, index, layerID).Scan(&ancestryLayerID); err != nil { + return nil, handleError("insertAncestryLayers", commonerr.CombineErrors(err, stmt.Close())) + } + + if !ancestryLayerID.Valid { + return nil, database.ErrInconsistent + } + + ancestryLayerIDs = append(ancestryLayerIDs, ancestryLayerID.Int64) + } + + if err := stmt.Close(); err != nil { + return nil, handleError("insertAncestryLayers", err) + } + + return ancestryLayerIDs, nil +} + +func (tx *pgSession) findAncestryLayers(id int64) ([]database.AncestryLayer, error) { + detectors, err := tx.findAllDetectors() + if err != nil { + return nil, err + } + + layerMap, err := tx.findAncestryLayerHashes(id) + if err != nil { + return nil, err + } + + featureMap, err := tx.findAncestryFeatures(id, detectors) + if err != nil { + return nil, err + } + + layers := make([]database.AncestryLayer, len(layerMap)) + for index, layer := range layerMap { + // index MUST match the ancestry layer slice index. + if layers[index].Hash == "" && len(layers[index].Features) == 0 { + layers[index] = database.AncestryLayer{ + Hash: layer, + Features: featureMap[index], + } + } else { + log.WithFields(log.Fields{ + "ancestry ID": id, + "duplicated ancestry index": index, + }).WithError(database.ErrInconsistent).Error("ancestry layers with same ancestry_index is not allowed") + return nil, database.ErrInconsistent + } + } + + return layers, nil +} diff --git a/database/pgsql/ancestry_test.go b/database/pgsql/ancestry/ancestry_test.go similarity index 100% rename from database/pgsql/ancestry_test.go rename to database/pgsql/ancestry/ancestry_test.go diff --git a/database/pgsql/detector.go b/database/pgsql/detector/detector.go similarity index 62% rename from database/pgsql/detector.go rename to database/pgsql/detector/detector.go index 3209d632d3..db4f39398a 100644 --- a/database/pgsql/detector.go +++ b/database/pgsql/detector/detector.go @@ -17,7 +17,6 @@ package pgsql import ( "database/sql" - "github.com/deckarep/golang-set" log "github.com/sirupsen/logrus" "github.com/coreos/clair/database" @@ -29,26 +28,6 @@ const ( SELECT CAST ($1 AS TEXT), CAST ($2 AS TEXT), CAST ($3 AS detector_type ) WHERE NOT EXISTS (SELECT id FROM detector WHERE name = $1 AND version = $2 AND dtype = $3);` - selectAncestryDetectors = ` - SELECT d.name, d.version, d.dtype - FROM ancestry_detector, detector AS d - WHERE ancestry_detector.detector_id = d.id AND ancestry_detector.ancestry_id = $1;` - - selectLayerDetectors = ` - SELECT d.name, d.version, d.dtype - FROM layer_detector, detector AS d - WHERE layer_detector.detector_id = d.id AND layer_detector.layer_id = $1;` - - insertAncestryDetectors = ` - INSERT INTO ancestry_detector (ancestry_id, detector_id) - SELECT $1, $2 - WHERE NOT EXISTS (SELECT id FROM ancestry_detector WHERE ancestry_id = $1 AND detector_id = $2)` - - persistLayerDetector = ` - INSERT INTO layer_detector (layer_id, detector_id) - SELECT $1, $2 - WHERE NOT EXISTS (SELECT id FROM layer_detector WHERE layer_id = $1 AND detector_id = $2)` - findDetectorID = `SELECT id FROM detector WHERE name = $1 AND version = $2 AND dtype = $3` findAllDetectors = `SELECT id, name, version, dtype FROM detector` ) @@ -83,50 +62,6 @@ func (tx *pgSession) PersistDetectors(detectors []database.Detector) error { return nil } -func (tx *pgSession) persistLayerDetector(layerID int64, detectorID int64) error { - if _, err := tx.Exec(persistLayerDetector, layerID, detectorID); err != nil { - return handleError("persistLayerDetector", err) - } - - return nil -} - -func (tx *pgSession) persistLayerDetectors(layerID int64, detectorIDs []int64) error { - alreadySaved := mapset.NewSet() - for _, id := range detectorIDs { - if alreadySaved.Contains(id) { - continue - } - - alreadySaved.Add(id) - if err := tx.persistLayerDetector(layerID, id); err != nil { - return err - } - } - - return nil -} - -func (tx *pgSession) insertAncestryDetectors(ancestryID int64, detectorIDs []int64) error { - for _, detectorID := range detectorIDs { - if _, err := tx.Exec(insertAncestryDetectors, ancestryID, detectorID); err != nil { - return handleError("insertAncestryDetectors", err) - } - } - - return nil -} - -func (tx *pgSession) findAncestryDetectors(id int64) ([]database.Detector, error) { - detectors, err := tx.getDetectors(selectAncestryDetectors, id) - return detectors, err -} - -func (tx *pgSession) findLayerDetectors(id int64) ([]database.Detector, error) { - detectors, err := tx.getDetectors(selectLayerDetectors, id) - return detectors, err -} - // findDetectorIDs retrieve ids of the detectors from the database, if any is not // found, return the error. func (tx *pgSession) findDetectorIDs(detectors []database.Detector) ([]int64, error) { diff --git a/database/pgsql/detector_test.go b/database/pgsql/detector/detector_test.go similarity index 100% rename from database/pgsql/detector_test.go rename to database/pgsql/detector/detector_test.go diff --git a/database/pgsql/entities/feature.go b/database/pgsql/entities/feature.go new file mode 100644 index 0000000000..32cef255a9 --- /dev/null +++ b/database/pgsql/entities/feature.go @@ -0,0 +1,101 @@ +// Copyright 2017 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pgsql + +import ( + "database/sql" + "sort" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/pkg/commonerr" +) + +func (tx *pgSession) PersistFeatures(features []database.Feature) error { + if len(features) == 0 { + return nil + } + + types, err := tx.getFeatureTypeMap() + if err != nil { + return err + } + + // Sorting is needed before inserting into database to prevent deadlock. + sort.Slice(features, func(i, j int) bool { + return features[i].Name < features[j].Name || + features[i].Version < features[j].Version || + features[i].VersionFormat < features[j].VersionFormat + }) + + // TODO(Sida): A better interface for bulk insertion is needed. + keys := make([]interface{}, 0, len(features)*3) + for _, f := range features { + keys = append(keys, f.Name, f.Version, f.VersionFormat, types.byName[f.Type]) + if f.Name == "" || f.Version == "" || f.VersionFormat == "" { + return commonerr.NewBadRequestError("Empty feature name, version or version format is not allowed") + } + } + + _, err = tx.Exec(queryPersistFeature(len(features)), keys...) + return handleError("queryPersistFeature", err) +} + +func (tx *pgSession) findFeatureIDs(fs []database.Feature) ([]sql.NullInt64, error) { + if len(fs) == 0 { + return nil, nil + } + + types, err := tx.getFeatureTypeMap() + if err != nil { + return nil, err + } + + fMap := map[database.Feature]sql.NullInt64{} + + keys := make([]interface{}, 0, len(fs)*4) + for _, f := range fs { + typeID := types.byName[f.Type] + keys = append(keys, f.Name, f.Version, f.VersionFormat, typeID) + fMap[f] = sql.NullInt64{} + } + + rows, err := tx.Query(querySearchFeatureID(len(fs)), keys...) + if err != nil { + return nil, handleError("querySearchFeatureID", err) + } + defer rows.Close() + + var ( + id sql.NullInt64 + f database.Feature + ) + for rows.Next() { + var typeID int + err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat, &typeID) + if err != nil { + return nil, handleError("querySearchFeatureID", err) + } + + f.Type = types.byID[typeID] + fMap[f] = id + } + + ids := make([]sql.NullInt64, len(fs)) + for i, f := range fs { + ids[i] = fMap[f] + } + + return ids, nil +} diff --git a/database/pgsql/feature_test.go b/database/pgsql/entities/feature_test.go similarity index 100% rename from database/pgsql/feature_test.go rename to database/pgsql/entities/feature_test.go diff --git a/database/pgsql/feature_type.go b/database/pgsql/entities/feature_type.go similarity index 100% rename from database/pgsql/feature_type.go rename to database/pgsql/entities/feature_type.go diff --git a/database/pgsql/feature_type_test.go b/database/pgsql/entities/feature_type_test.go similarity index 100% rename from database/pgsql/feature_type_test.go rename to database/pgsql/entities/feature_type_test.go diff --git a/database/pgsql/namespace.go b/database/pgsql/entities/namespace.go similarity index 100% rename from database/pgsql/namespace.go rename to database/pgsql/entities/namespace.go diff --git a/database/pgsql/namespace_test.go b/database/pgsql/entities/namespace_test.go similarity index 100% rename from database/pgsql/namespace_test.go rename to database/pgsql/entities/namespace_test.go diff --git a/database/pgsql/entities/namespaced_feature.go b/database/pgsql/entities/namespaced_feature.go new file mode 100644 index 0000000000..ed0d3c99bb --- /dev/null +++ b/database/pgsql/entities/namespaced_feature.go @@ -0,0 +1,131 @@ +package pgsql + +import ( + "database/sql" + "sort" + + "github.com/coreos/clair/database" +) + +var soiNamespacedFeature = ` +WITH new_feature_ns AS ( + INSERT INTO namespaced_feature(feature_id, namespace_id) + SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER) + WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2) + RETURNING id +) +SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2 +UNION +SELECT id FROM new_feature_ns` + +type namespacedFeatureWithID struct { + database.NamespacedFeature + + ID int64 +} + +func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFeature) error { + if len(features) == 0 { + return nil + } + + nsIDs := map[database.Namespace]sql.NullInt64{} + fIDs := map[database.Feature]sql.NullInt64{} + for _, f := range features { + nsIDs[f.Namespace] = sql.NullInt64{} + fIDs[f.Feature] = sql.NullInt64{} + } + + fToFind := []database.Feature{} + for f := range fIDs { + fToFind = append(fToFind, f) + } + + sort.Slice(fToFind, func(i, j int) bool { + return fToFind[i].Name < fToFind[j].Name || + fToFind[i].Version < fToFind[j].Version || + fToFind[i].VersionFormat < fToFind[j].VersionFormat + }) + + if ids, err := tx.findFeatureIDs(fToFind); err == nil { + for i, id := range ids { + if !id.Valid { + return database.ErrMissingEntities + } + fIDs[fToFind[i]] = id + } + } else { + return err + } + + nsToFind := []database.Namespace{} + for ns := range nsIDs { + nsToFind = append(nsToFind, ns) + } + + if ids, err := tx.findNamespaceIDs(nsToFind); err == nil { + for i, id := range ids { + if !id.Valid { + return database.ErrMissingEntities + } + nsIDs[nsToFind[i]] = id + } + } else { + return err + } + + keys := make([]interface{}, 0, len(features)*2) + for _, f := range features { + keys = append(keys, fIDs[f.Feature], nsIDs[f.Namespace]) + } + + _, err := tx.Exec(queryPersistNamespacedFeature(len(features)), keys...) + if err != nil { + return err + } + + return nil +} + +func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) ([]sql.NullInt64, error) { + if len(nfs) == 0 { + return nil, nil + } + + nfsMap := map[database.NamespacedFeature]int64{} + keys := make([]interface{}, 0, len(nfs)*5) + for _, nf := range nfs { + keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Type, nf.Namespace.Name) + } + + rows, err := tx.Query(querySearchNamespacedFeature(len(nfs)), keys...) + if err != nil { + return nil, handleError("searchNamespacedFeature", err) + } + + defer rows.Close() + var ( + id int64 + nf database.NamespacedFeature + ) + + for rows.Next() { + err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Type, &nf.Namespace.Name) + nf.Namespace.VersionFormat = nf.VersionFormat + if err != nil { + return nil, handleError("searchNamespacedFeature", err) + } + nfsMap[nf] = id + } + + ids := make([]sql.NullInt64, len(nfs)) + for i, nf := range nfs { + if id, ok := nfsMap[nf]; ok { + ids[i] = sql.NullInt64{id, true} + } else { + ids[i] = sql.NullInt64{} + } + } + + return ids, nil +} diff --git a/database/pgsql/feature.go b/database/pgsql/feature.go deleted file mode 100644 index 66b47c507f..0000000000 --- a/database/pgsql/feature.go +++ /dev/null @@ -1,398 +0,0 @@ -// Copyright 2017 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pgsql - -import ( - "database/sql" - "sort" - - "github.com/lib/pq" - log "github.com/sirupsen/logrus" - - "github.com/coreos/clair/database" - "github.com/coreos/clair/ext/versionfmt" - "github.com/coreos/clair/pkg/commonerr" -) - -const ( - soiNamespacedFeature = ` - WITH new_feature_ns AS ( - INSERT INTO namespaced_feature(feature_id, namespace_id) - SELECT CAST ($1 AS INTEGER), CAST ($2 AS INTEGER) - WHERE NOT EXISTS ( SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2) - RETURNING id - ) - SELECT id FROM namespaced_feature WHERE namespaced_feature.feature_id = $1 AND namespaced_feature.namespace_id = $2 - UNION - SELECT id FROM new_feature_ns` - - searchPotentialAffectingVulneraibilities = ` - SELECT nf.id, v.id, vaf.affected_version, vaf.id - FROM vulnerability_affected_feature AS vaf, vulnerability AS v, - namespaced_feature AS nf, feature AS f - WHERE nf.id = ANY($1) - AND nf.feature_id = f.id - AND nf.namespace_id = v.namespace_id - AND vaf.feature_name = f.name - AND vaf.feature_type = f.type - AND vaf.vulnerability_id = v.id - AND v.deleted_at IS NULL` - - searchNamespacedFeaturesVulnerabilities = ` - SELECT vanf.namespaced_feature_id, v.name, v.description, v.link, - v.severity, v.metadata, vaf.fixedin, n.name, n.version_format - FROM vulnerability_affected_namespaced_feature AS vanf, - Vulnerability AS v, - vulnerability_affected_feature AS vaf, - namespace AS n - WHERE vanf.namespaced_feature_id = ANY($1) - AND vaf.id = vanf.added_by - AND v.id = vanf.vulnerability_id - AND n.id = v.namespace_id - AND v.deleted_at IS NULL` -) - -func (tx *pgSession) PersistFeatures(features []database.Feature) error { - if len(features) == 0 { - return nil - } - - types, err := tx.getFeatureTypeMap() - if err != nil { - return err - } - - // Sorting is needed before inserting into database to prevent deadlock. - sort.Slice(features, func(i, j int) bool { - return features[i].Name < features[j].Name || - features[i].Version < features[j].Version || - features[i].VersionFormat < features[j].VersionFormat - }) - - // TODO(Sida): A better interface for bulk insertion is needed. - keys := make([]interface{}, 0, len(features)*3) - for _, f := range features { - keys = append(keys, f.Name, f.Version, f.VersionFormat, types.byName[f.Type]) - if f.Name == "" || f.Version == "" || f.VersionFormat == "" { - return commonerr.NewBadRequestError("Empty feature name, version or version format is not allowed") - } - } - - _, err = tx.Exec(queryPersistFeature(len(features)), keys...) - return handleError("queryPersistFeature", err) -} - -type namespacedFeatureWithID struct { - database.NamespacedFeature - - ID int64 -} - -type vulnerabilityCache struct { - nsFeatureID int64 - vulnID int64 - vulnAffectingID int64 -} - -func (tx *pgSession) searchAffectingVulnerabilities(features []database.NamespacedFeature) ([]vulnerabilityCache, error) { - if len(features) == 0 { - return nil, nil - } - - ids, err := tx.findNamespacedFeatureIDs(features) - if err != nil { - return nil, err - } - - fMap := map[int64]database.NamespacedFeature{} - for i, f := range features { - if !ids[i].Valid { - return nil, database.ErrMissingEntities - } - fMap[ids[i].Int64] = f - } - - cacheTable := []vulnerabilityCache{} - rows, err := tx.Query(searchPotentialAffectingVulneraibilities, pq.Array(ids)) - if err != nil { - return nil, handleError("searchPotentialAffectingVulneraibilities", err) - } - - defer rows.Close() - for rows.Next() { - var ( - cache vulnerabilityCache - affected string - ) - - err := rows.Scan(&cache.nsFeatureID, &cache.vulnID, &affected, &cache.vulnAffectingID) - if err != nil { - return nil, err - } - - if ok, err := versionfmt.InRange(fMap[cache.nsFeatureID].VersionFormat, fMap[cache.nsFeatureID].Version, affected); err != nil { - return nil, err - } else if ok { - cacheTable = append(cacheTable, cache) - } - } - - return cacheTable, nil -} - -func (tx *pgSession) CacheAffectedNamespacedFeatures(features []database.NamespacedFeature) error { - if len(features) == 0 { - return nil - } - - _, err := tx.Exec(lockVulnerabilityAffects) - if err != nil { - return handleError("lockVulnerabilityAffects", err) - } - - cache, err := tx.searchAffectingVulnerabilities(features) - if err != nil { - return err - } - - keys := make([]interface{}, 0, len(cache)*3) - for _, c := range cache { - keys = append(keys, c.vulnID, c.nsFeatureID, c.vulnAffectingID) - } - - if len(cache) == 0 { - return nil - } - - affected, err := tx.Exec(queryPersistVulnerabilityAffectedNamespacedFeature(len(cache)), keys...) - if err != nil { - return handleError("persistVulnerabilityAffectedNamespacedFeature", err) - } - if count, err := affected.RowsAffected(); err != nil { - log.Debugf("Cached %d features in vulnerability_affected_namespaced_feature", count) - } - return nil -} - -func (tx *pgSession) PersistNamespacedFeatures(features []database.NamespacedFeature) error { - if len(features) == 0 { - return nil - } - - nsIDs := map[database.Namespace]sql.NullInt64{} - fIDs := map[database.Feature]sql.NullInt64{} - for _, f := range features { - nsIDs[f.Namespace] = sql.NullInt64{} - fIDs[f.Feature] = sql.NullInt64{} - } - - fToFind := []database.Feature{} - for f := range fIDs { - fToFind = append(fToFind, f) - } - - sort.Slice(fToFind, func(i, j int) bool { - return fToFind[i].Name < fToFind[j].Name || - fToFind[i].Version < fToFind[j].Version || - fToFind[i].VersionFormat < fToFind[j].VersionFormat - }) - - if ids, err := tx.findFeatureIDs(fToFind); err == nil { - for i, id := range ids { - if !id.Valid { - return database.ErrMissingEntities - } - fIDs[fToFind[i]] = id - } - } else { - return err - } - - nsToFind := []database.Namespace{} - for ns := range nsIDs { - nsToFind = append(nsToFind, ns) - } - - if ids, err := tx.findNamespaceIDs(nsToFind); err == nil { - for i, id := range ids { - if !id.Valid { - return database.ErrMissingEntities - } - nsIDs[nsToFind[i]] = id - } - } else { - return err - } - - keys := make([]interface{}, 0, len(features)*2) - for _, f := range features { - keys = append(keys, fIDs[f.Feature], nsIDs[f.Namespace]) - } - - _, err := tx.Exec(queryPersistNamespacedFeature(len(features)), keys...) - if err != nil { - return err - } - - return nil -} - -// FindAffectedNamespacedFeatures retrieves vulnerabilities associated with the -// feature. -func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.NamespacedFeature) ([]database.NullableAffectedNamespacedFeature, error) { - if len(features) == 0 { - return nil, nil - } - - vulnerableFeatures := make([]database.NullableAffectedNamespacedFeature, len(features)) - featureIDs, err := tx.findNamespacedFeatureIDs(features) - if err != nil { - return nil, err - } - - for i, id := range featureIDs { - if id.Valid { - vulnerableFeatures[i].Valid = true - vulnerableFeatures[i].NamespacedFeature = features[i] - } - } - - rows, err := tx.Query(searchNamespacedFeaturesVulnerabilities, pq.Array(featureIDs)) - if err != nil { - return nil, handleError("searchNamespacedFeaturesVulnerabilities", err) - } - defer rows.Close() - - for rows.Next() { - var ( - featureID int64 - vuln database.VulnerabilityWithFixedIn - ) - - err := rows.Scan(&featureID, - &vuln.Name, - &vuln.Description, - &vuln.Link, - &vuln.Severity, - &vuln.Metadata, - &vuln.FixedInVersion, - &vuln.Namespace.Name, - &vuln.Namespace.VersionFormat, - ) - - if err != nil { - return nil, handleError("searchNamespacedFeaturesVulnerabilities", err) - } - - for i, id := range featureIDs { - if id.Valid && id.Int64 == featureID { - vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy = append(vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy, vuln) - } - } - } - - return vulnerableFeatures, nil -} - -func (tx *pgSession) findNamespacedFeatureIDs(nfs []database.NamespacedFeature) ([]sql.NullInt64, error) { - if len(nfs) == 0 { - return nil, nil - } - - nfsMap := map[database.NamespacedFeature]int64{} - keys := make([]interface{}, 0, len(nfs)*5) - for _, nf := range nfs { - keys = append(keys, nf.Name, nf.Version, nf.VersionFormat, nf.Type, nf.Namespace.Name) - } - - rows, err := tx.Query(querySearchNamespacedFeature(len(nfs)), keys...) - if err != nil { - return nil, handleError("searchNamespacedFeature", err) - } - - defer rows.Close() - var ( - id int64 - nf database.NamespacedFeature - ) - - for rows.Next() { - err := rows.Scan(&id, &nf.Name, &nf.Version, &nf.VersionFormat, &nf.Type, &nf.Namespace.Name) - nf.Namespace.VersionFormat = nf.VersionFormat - if err != nil { - return nil, handleError("searchNamespacedFeature", err) - } - nfsMap[nf] = id - } - - ids := make([]sql.NullInt64, len(nfs)) - for i, nf := range nfs { - if id, ok := nfsMap[nf]; ok { - ids[i] = sql.NullInt64{id, true} - } else { - ids[i] = sql.NullInt64{} - } - } - - return ids, nil -} - -func (tx *pgSession) findFeatureIDs(fs []database.Feature) ([]sql.NullInt64, error) { - if len(fs) == 0 { - return nil, nil - } - - types, err := tx.getFeatureTypeMap() - if err != nil { - return nil, err - } - - fMap := map[database.Feature]sql.NullInt64{} - - keys := make([]interface{}, 0, len(fs)*4) - for _, f := range fs { - typeID := types.byName[f.Type] - keys = append(keys, f.Name, f.Version, f.VersionFormat, typeID) - fMap[f] = sql.NullInt64{} - } - - rows, err := tx.Query(querySearchFeatureID(len(fs)), keys...) - if err != nil { - return nil, handleError("querySearchFeatureID", err) - } - defer rows.Close() - - var ( - id sql.NullInt64 - f database.Feature - ) - for rows.Next() { - var typeID int - err := rows.Scan(&id, &f.Name, &f.Version, &f.VersionFormat, &typeID) - if err != nil { - return nil, handleError("querySearchFeatureID", err) - } - - f.Type = types.byID[typeID] - fMap[f] = id - } - - ids := make([]sql.NullInt64, len(fs)) - for i, f := range fs { - ids[i] = fMap[f] - } - - return ids, nil -} diff --git a/database/pgsql/layer.go b/database/pgsql/layer/layer.go similarity index 78% rename from database/pgsql/layer.go rename to database/pgsql/layer/layer.go index a071eb4b47..e69dfbbf5e 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer/layer.go @@ -36,36 +36,9 @@ const ( UNION SELECT id FROM layer WHERE hash = $1` - findLayerFeatures = ` - SELECT f.name, f.version, f.version_format, t.name, lf.detector_id - FROM layer_feature AS lf, feature AS f, feature_type AS t - WHERE lf.feature_id = f.id - AND t.id = f.type - AND lf.layer_id = $1` - - findLayerNamespaces = ` - SELECT ns.name, ns.version_format, ln.detector_id - FROM layer_namespace AS ln, namespace AS ns - WHERE ln.namespace_id = ns.id - AND ln.layer_id = $1` - findLayerID = `SELECT id FROM layer WHERE hash = $1` ) -// dbLayerNamespace represents the layer_namespace table. -type dbLayerNamespace struct { - layerID int64 - namespaceID int64 - detectorID int64 -} - -// dbLayerFeature represents the layer_feature table -type dbLayerFeature struct { - layerID int64 - featureID int64 - detectorID int64 -} - func (tx *pgSession) FindLayer(hash string) (database.Layer, bool, error) { layer := database.Layer{Hash: hash} if hash == "" { @@ -271,54 +244,6 @@ func (tx *pgSession) persistLayerNamespaces(namespaces []dbLayerNamespace) error return nil } -func (tx *pgSession) findLayerNamespaces(layerID int64, detectors detectorMap) ([]database.LayerNamespace, error) { - rows, err := tx.Query(findLayerNamespaces, layerID) - if err != nil { - return nil, handleError("findLayerNamespaces", err) - } - - namespaces := []database.LayerNamespace{} - for rows.Next() { - var ( - namespace database.LayerNamespace - detectorID int64 - ) - - if err := rows.Scan(&namespace.Name, &namespace.VersionFormat, &detectorID); err != nil { - return nil, err - } - - namespace.By = detectors.byID[detectorID] - namespaces = append(namespaces, namespace) - } - - return namespaces, nil -} - -func (tx *pgSession) findLayerFeatures(layerID int64, detectors detectorMap) ([]database.LayerFeature, error) { - rows, err := tx.Query(findLayerFeatures, layerID) - if err != nil { - return nil, handleError("findLayerFeatures", err) - } - defer rows.Close() - - features := []database.LayerFeature{} - for rows.Next() { - var ( - detectorID int64 - feature database.LayerFeature - ) - if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &feature.Type, &detectorID); err != nil { - return nil, handleError("findLayerFeatures", err) - } - - feature.By = detectors.byID[detectorID] - features = append(features, feature) - } - - return features, nil -} - func (tx *pgSession) findLayerID(hash string) (int64, bool, error) { var layerID int64 err := tx.QueryRow(findLayerID, hash).Scan(&layerID) diff --git a/database/pgsql/layer/layer_detector.go b/database/pgsql/layer/layer_detector.go new file mode 100644 index 0000000000..0ac0a4d1b6 --- /dev/null +++ b/database/pgsql/layer/layer_detector.go @@ -0,0 +1,48 @@ +package pgsql + +import ( + "github.com/deckarep/golang-set" + + "github.com/coreos/clair/database" +) + +const ( + selectLayerDetectors = ` + SELECT d.name, d.version, d.dtype + FROM layer_detector, detector AS d + WHERE layer_detector.detector_id = d.id AND layer_detector.layer_id = $1;` + + persistLayerDetector = ` + INSERT INTO layer_detector (layer_id, detector_id) + SELECT $1, $2 + WHERE NOT EXISTS (SELECT id FROM layer_detector WHERE layer_id = $1 AND detector_id = $2)` +) + +func (tx *pgSession) persistLayerDetector(layerID int64, detectorID int64) error { + if _, err := tx.Exec(persistLayerDetector, layerID, detectorID); err != nil { + return handleError("persistLayerDetector", err) + } + + return nil +} + +func (tx *pgSession) persistLayerDetectors(layerID int64, detectorIDs []int64) error { + alreadySaved := mapset.NewSet() + for _, id := range detectorIDs { + if alreadySaved.Contains(id) { + continue + } + + alreadySaved.Add(id) + if err := tx.persistLayerDetector(layerID, id); err != nil { + return err + } + } + + return nil +} + +func (tx *pgSession) findLayerDetectors(id int64) ([]database.Detector, error) { + detectors, err := tx.getDetectors(selectLayerDetectors, id) + return detectors, err +} diff --git a/database/pgsql/layer/layer_feature.go b/database/pgsql/layer/layer_feature.go new file mode 100644 index 0000000000..7ff4019132 --- /dev/null +++ b/database/pgsql/layer/layer_feature.go @@ -0,0 +1,41 @@ +package pgsql + +import "github.com/coreos/clair/database" + +const findLayerFeatures = ` +SELECT f.name, f.version, f.version_format, t.name, lf.detector_id + FROM layer_feature AS lf, feature AS f, feature_type AS t + WHERE lf.feature_id = f.id + AND t.id = f.type + AND lf.layer_id = $1` + +// dbLayerFeature represents the layer_feature table +type dbLayerFeature struct { + layerID int64 + featureID int64 + detectorID int64 +} + +func (tx *pgSession) findLayerFeatures(layerID int64, detectors detectorMap) ([]database.LayerFeature, error) { + rows, err := tx.Query(findLayerFeatures, layerID) + if err != nil { + return nil, handleError("findLayerFeatures", err) + } + defer rows.Close() + + features := []database.LayerFeature{} + for rows.Next() { + var ( + detectorID int64 + feature database.LayerFeature + ) + if err := rows.Scan(&feature.Name, &feature.Version, &feature.VersionFormat, &feature.Type, &detectorID); err != nil { + return nil, handleError("findLayerFeatures", err) + } + + feature.By = detectors.byID[detectorID] + features = append(features, feature) + } + + return features, nil +} diff --git a/database/pgsql/layer/layer_namespace.go b/database/pgsql/layer/layer_namespace.go new file mode 100644 index 0000000000..b8dcc4c52b --- /dev/null +++ b/database/pgsql/layer/layer_namespace.go @@ -0,0 +1,40 @@ +package pgsql + +import "github.com/coreos/clair/database" + +const findLayerNamespaces = ` +SELECT ns.name, ns.version_format, ln.detector_id + FROM layer_namespace AS ln, namespace AS ns + WHERE ln.namespace_id = ns.id + AND ln.layer_id = $1` + +// dbLayerNamespace represents the layer_namespace table. +type dbLayerNamespace struct { + layerID int64 + namespaceID int64 + detectorID int64 +} + +func (tx *pgSession) findLayerNamespaces(layerID int64, detectors detectorMap) ([]database.LayerNamespace, error) { + rows, err := tx.Query(findLayerNamespaces, layerID) + if err != nil { + return nil, handleError("findLayerNamespaces", err) + } + + namespaces := []database.LayerNamespace{} + for rows.Next() { + var ( + namespace database.LayerNamespace + detectorID int64 + ) + + if err := rows.Scan(&namespace.Name, &namespace.VersionFormat, &detectorID); err != nil { + return nil, err + } + + namespace.By = detectors.byID[detectorID] + namespaces = append(namespaces, namespace) + } + + return namespaces, nil +} diff --git a/database/pgsql/layer_test.go b/database/pgsql/layer/layer_test.go similarity index 100% rename from database/pgsql/layer_test.go rename to database/pgsql/layer/layer_test.go diff --git a/database/pgsql/keyvalue.go b/database/pgsql/lock/keyvalue.go similarity index 100% rename from database/pgsql/keyvalue.go rename to database/pgsql/lock/keyvalue.go diff --git a/database/pgsql/keyvalue_test.go b/database/pgsql/lock/keyvalue_test.go similarity index 100% rename from database/pgsql/keyvalue_test.go rename to database/pgsql/lock/keyvalue_test.go diff --git a/database/pgsql/lock.go b/database/pgsql/lock/lock.go similarity index 100% rename from database/pgsql/lock.go rename to database/pgsql/lock/lock.go diff --git a/database/pgsql/lock_test.go b/database/pgsql/lock/lock_test.go similarity index 100% rename from database/pgsql/lock_test.go rename to database/pgsql/lock/lock_test.go diff --git a/database/pgsql/migrations_test.go b/database/pgsql/migrations/migrations_test.go similarity index 100% rename from database/pgsql/migrations_test.go rename to database/pgsql/migrations/migrations_test.go diff --git a/database/pgsql/notification_test.go b/database/pgsql/notification/notification_test.go similarity index 100% rename from database/pgsql/notification_test.go rename to database/pgsql/notification/notification_test.go diff --git a/database/pgsql/notification.go b/database/pgsql/notification/vulnerability_notification.go similarity index 96% rename from database/pgsql/notification.go rename to database/pgsql/notification/vulnerability_notification.go index 7d2b750df9..24bb86d938 100644 --- a/database/pgsql/notification.go +++ b/database/pgsql/notification/vulnerability_notification.go @@ -56,17 +56,17 @@ const ( WHERE name = $1` searchNotificationVulnerableAncestry = ` - SELECT DISTINCT ON (a.id) - a.id, a.name - FROM vulnerability_affected_namespaced_feature AS vanf, - ancestry_layer AS al, ancestry_feature AS af, ancestry AS a - WHERE vanf.vulnerability_id = $1 - AND a.id >= $2 - AND al.ancestry_id = a.id - AND al.id = af.ancestry_layer_id - AND af.namespaced_feature_id = vanf.namespaced_feature_id - ORDER BY a.id ASC - LIMIT $3;` + SELECT DISTINCT ON (a.id) + a.id, a.name + FROM vulnerability_affected_namespaced_feature AS vanf, + ancestry_layer AS al, ancestry_feature AS af, ancestry AS a + WHERE vanf.vulnerability_id = $1 + AND a.id >= $2 + AND al.ancestry_id = a.id + AND al.id = af.ancestry_layer_id + AND af.namespaced_feature_id = vanf.namespaced_feature_id + ORDER BY a.id ASC + LIMIT $3;` ) var ( diff --git a/database/pgsql/testdata/data.sql b/database/pgsql/testutil/data.sql similarity index 100% rename from database/pgsql/testdata/data.sql rename to database/pgsql/testutil/data.sql diff --git a/database/pgsql/testutil.go b/database/pgsql/testutil/testutil.go similarity index 100% rename from database/pgsql/testutil.go rename to database/pgsql/testutil/testutil.go diff --git a/database/pgsql/queries.go b/database/pgsql/util/queries.go similarity index 100% rename from database/pgsql/queries.go rename to database/pgsql/util/queries.go diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability/vulnerability.go similarity index 90% rename from database/pgsql/vulnerability.go rename to database/pgsql/vulnerability/vulnerability.go index e96d6d476f..6921639060 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability/vulnerability.go @@ -27,8 +27,6 @@ import ( ) const ( - lockVulnerabilityAffects = `LOCK vulnerability_affected_namespaced_feature IN SHARE ROW EXCLUSIVE MODE` - searchVulnerability = ` SELECT v.id, v.description, v.link, v.severity, v.metadata, n.version_format FROM vulnerability AS v, namespace AS n @@ -38,45 +36,12 @@ const ( AND v.deleted_at IS NULL ` - insertVulnerabilityAffected = ` - INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, feature_type, fixedin) - VALUES ($1, $2, $3, $4, $5) - RETURNING ID - ` - - searchVulnerabilityAffected = ` - SELECT vulnerability_id, feature_name, affected_version, t.name, fixedin - FROM vulnerability_affected_feature AS vaf, feature_type AS t - WHERE t.id = vaf.feature_type AND vulnerability_id = ANY($1) - ` - searchVulnerabilityByID = ` SELECT v.name, v.description, v.link, v.severity, v.metadata, n.name, n.version_format FROM vulnerability AS v, namespace AS n WHERE v.namespace_id = n.id AND v.id = $1` - searchVulnerabilityPotentialAffected = ` - WITH req AS ( - SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, vaf.feature_type AS type, v.id AS vulnerability_id - FROM vulnerability_affected_feature AS vaf, - vulnerability AS v, - namespace AS n - WHERE vaf.vulnerability_id = ANY($1) - AND v.id = vaf.vulnerability_id - AND n.id = v.namespace_id - ) - SELECT req.vulnerability_id, nf.id, f.version, req.vaf_id AS added_by - FROM feature AS f, namespaced_feature AS nf, req - WHERE f.name = req.name - AND f.type = req.type - AND nf.namespace_id = req.n_id - AND nf.feature_id = f.id` - - insertVulnerabilityAffectedNamespacedFeature = ` - INSERT INTO vulnerability_affected_namespaced_feature(vulnerability_id, namespaced_feature_id, added_by) - VALUES ($1, $2, $3)` - insertVulnerability = ` WITH ns AS ( SELECT id FROM namespace WHERE name = $6 AND version_format = $7 diff --git a/database/pgsql/vulnerability/vulnerability_affected_feature.go b/database/pgsql/vulnerability/vulnerability_affected_feature.go new file mode 100644 index 0000000000..37069f8c93 --- /dev/null +++ b/database/pgsql/vulnerability/vulnerability_affected_feature.go @@ -0,0 +1,100 @@ +package pgsql + +import ( + "github.com/coreos/clair/database" + "github.com/coreos/clair/ext/versionfmt" + "github.com/lib/pq" +) + +const ( + searchPotentialAffectingVulneraibilities = ` + SELECT nf.id, v.id, vaf.affected_version, vaf.id + FROM vulnerability_affected_feature AS vaf, vulnerability AS v, + namespaced_feature AS nf, feature AS f + WHERE nf.id = ANY($1) + AND nf.feature_id = f.id + AND nf.namespace_id = v.namespace_id + AND vaf.feature_name = f.name + AND vaf.feature_type = f.type + AND vaf.vulnerability_id = v.id + AND v.deleted_at IS NULL` + insertVulnerabilityAffected = ` + INSERT INTO vulnerability_affected_feature(vulnerability_id, feature_name, affected_version, feature_type, fixedin) + VALUES ($1, $2, $3, $4, $5) + RETURNING ID + ` + searchVulnerabilityAffected = ` + SELECT vulnerability_id, feature_name, affected_version, t.name, fixedin + FROM vulnerability_affected_feature AS vaf, feature_type AS t + WHERE t.id = vaf.feature_type AND vulnerability_id = ANY($1) + ` + + searchVulnerabilityPotentialAffected = ` + WITH req AS ( + SELECT vaf.id AS vaf_id, n.id AS n_id, vaf.feature_name AS name, vaf.feature_type AS type, v.id AS vulnerability_id + FROM vulnerability_affected_feature AS vaf, + vulnerability AS v, + namespace AS n + WHERE vaf.vulnerability_id = ANY($1) + AND v.id = vaf.vulnerability_id + AND n.id = v.namespace_id + ) + SELECT req.vulnerability_id, nf.id, f.version, req.vaf_id AS added_by + FROM feature AS f, namespaced_feature AS nf, req + WHERE f.name = req.name + AND f.type = req.type + AND nf.namespace_id = req.n_id + AND nf.feature_id = f.id` +) + +type vulnerabilityCache struct { + nsFeatureID int64 + vulnID int64 + vulnAffectingID int64 +} + +func (tx *pgSession) searchAffectingVulnerabilities(features []database.NamespacedFeature) ([]vulnerabilityCache, error) { + if len(features) == 0 { + return nil, nil + } + + ids, err := tx.findNamespacedFeatureIDs(features) + if err != nil { + return nil, err + } + + fMap := map[int64]database.NamespacedFeature{} + for i, f := range features { + if !ids[i].Valid { + return nil, database.ErrMissingEntities + } + fMap[ids[i].Int64] = f + } + + cacheTable := []vulnerabilityCache{} + rows, err := tx.Query(searchPotentialAffectingVulneraibilities, pq.Array(ids)) + if err != nil { + return nil, handleError("searchPotentialAffectingVulneraibilities", err) + } + + defer rows.Close() + for rows.Next() { + var ( + cache vulnerabilityCache + affected string + ) + + err := rows.Scan(&cache.nsFeatureID, &cache.vulnID, &affected, &cache.vulnAffectingID) + if err != nil { + return nil, err + } + + if ok, err := versionfmt.InRange(fMap[cache.nsFeatureID].VersionFormat, fMap[cache.nsFeatureID].Version, affected); err != nil { + return nil, err + } else if ok { + cacheTable = append(cacheTable, cache) + } + } + + return cacheTable, nil +} diff --git a/database/pgsql/vulnerability/vulnerability_affected_namespaced_feature.go b/database/pgsql/vulnerability/vulnerability_affected_namespaced_feature.go new file mode 100644 index 0000000000..abf70f53ed --- /dev/null +++ b/database/pgsql/vulnerability/vulnerability_affected_namespaced_feature.go @@ -0,0 +1,116 @@ +package pgsql + +import ( + "github.com/coreos/clair/database" + "github.com/lib/pq" + log "github.com/sirupsen/logrus" +) + +const ( + searchNamespacedFeaturesVulnerabilities = ` + SELECT vanf.namespaced_feature_id, v.name, v.description, v.link, + v.severity, v.metadata, vaf.fixedin, n.name, n.version_format + FROM vulnerability_affected_namespaced_feature AS vanf, + Vulnerability AS v, + vulnerability_affected_feature AS vaf, + namespace AS n + WHERE vanf.namespaced_feature_id = ANY($1) + AND vaf.id = vanf.added_by + AND v.id = vanf.vulnerability_id + AND n.id = v.namespace_id + AND v.deleted_at IS NULL` + + lockVulnerabilityAffects = `LOCK vulnerability_affected_namespaced_feature IN SHARE ROW EXCLUSIVE MODE` + + insertVulnerabilityAffectedNamespacedFeature = ` + INSERT INTO vulnerability_affected_namespaced_feature(vulnerability_id, namespaced_feature_id, added_by) + VALUES ($1, $2, $3)` +) + +// FindAffectedNamespacedFeatures retrieves vulnerabilities associated with the +// feature. +func (tx *pgSession) FindAffectedNamespacedFeatures(features []database.NamespacedFeature) ([]database.NullableAffectedNamespacedFeature, error) { + if len(features) == 0 { + return nil, nil + } + + vulnerableFeatures := make([]database.NullableAffectedNamespacedFeature, len(features)) + featureIDs, err := tx.findNamespacedFeatureIDs(features) + if err != nil { + return nil, err + } + + for i, id := range featureIDs { + if id.Valid { + vulnerableFeatures[i].Valid = true + vulnerableFeatures[i].NamespacedFeature = features[i] + } + } + + rows, err := tx.Query(searchNamespacedFeaturesVulnerabilities, pq.Array(featureIDs)) + if err != nil { + return nil, handleError("searchNamespacedFeaturesVulnerabilities", err) + } + defer rows.Close() + + for rows.Next() { + var ( + featureID int64 + vuln database.VulnerabilityWithFixedIn + ) + + err := rows.Scan(&featureID, + &vuln.Name, + &vuln.Description, + &vuln.Link, + &vuln.Severity, + &vuln.Metadata, + &vuln.FixedInVersion, + &vuln.Namespace.Name, + &vuln.Namespace.VersionFormat, + ) + + if err != nil { + return nil, handleError("searchNamespacedFeaturesVulnerabilities", err) + } + + for i, id := range featureIDs { + if id.Valid && id.Int64 == featureID { + vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy = append(vulnerableFeatures[i].AffectedNamespacedFeature.AffectedBy, vuln) + } + } + } + + return vulnerableFeatures, nil +} + +func (tx *pgSession) CacheAffectedNamespacedFeatures(features []database.NamespacedFeature) error { + if len(features) == 0 { + return nil + } + + _, err := tx.Exec(lockVulnerabilityAffects) + if err != nil { + return handleError("lockVulnerabilityAffects", err) + } + + cache, err := tx.searchAffectingVulnerabilities(features) + + keys := make([]interface{}, 0, len(cache)*3) + for _, c := range cache { + keys = append(keys, c.vulnID, c.nsFeatureID, c.vulnAffectingID) + } + + if len(cache) == 0 { + return nil + } + + affected, err := tx.Exec(queryPersistVulnerabilityAffectedNamespacedFeature(len(cache)), keys...) + if err != nil { + return handleError("persistVulnerabilityAffectedNamespacedFeature", err) + } + if count, err := affected.RowsAffected(); err != nil { + log.Debugf("Cached %d features in vulnerability_affected_namespaced_feature", count) + } + return nil +} diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability/vulnerability_test.go similarity index 100% rename from database/pgsql/vulnerability_test.go rename to database/pgsql/vulnerability/vulnerability_test.go diff --git a/database/vulnerability.go b/database/vulnerability.go new file mode 100644 index 0000000000..91f6c027d5 --- /dev/null +++ b/database/vulnerability.go @@ -0,0 +1,36 @@ +package database + +// VulnerabilityID is an identifier for every vulnerability. Every vulnerability +// has unique namespace and name. +type VulnerabilityID struct { + Name string + Namespace string +} + +// Vulnerability represents CVE or similar vulnerability reports. +type Vulnerability struct { + Name string + Namespace Namespace + + Description string + Link string + Severity Severity + + Metadata MetadataMap +} + +// VulnerabilityWithAffected is a vulnerability with all known affected +// features. +type VulnerabilityWithAffected struct { + Vulnerability + + Affected []AffectedFeature +} + +// NullableVulnerability is a vulnerability with whether the vulnerability is +// found in datastore. +type NullableVulnerability struct { + VulnerabilityWithAffected + + Valid bool +}