Skip to content

Commit

Permalink
*: add metadata support along with NVD CVSS
Browse files Browse the repository at this point in the history
  • Loading branch information
Quentin-M authored and jzelinskie committed Feb 24, 2016
1 parent c05848e commit 5fdd9d1
Show file tree
Hide file tree
Showing 17 changed files with 522 additions and 264 deletions.
18 changes: 11 additions & 7 deletions api/v1/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil
Namespace: dbVuln.Namespace.Name,
Description: dbVuln.Description,
Severity: string(dbVuln.Severity),
Metadata: dbVuln.Metadata,
}

if dbVuln.FixedBy != types.MaxVersion {
Expand All @@ -78,13 +79,14 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil
}

type Vulnerability struct {
Name string `json:"Name,omitempty"`
Namespace string `json:"Namespace,omitempty"`
Description string `json:"Description,omitempty"`
Link string `json:"Link,omitempty"`
Severity string `json:"Severity,omitempty"`
FixedBy string `json:"FixedBy,omitempty"`
FixedIn []Feature `json:"FixedIn,omitempty"`
Name string `json:"Name,omitempty"`
Namespace string `json:"Namespace,omitempty"`
Description string `json:"Description,omitempty"`
Link string `json:"Link,omitempty"`
Severity string `json:"Severity,omitempty"`
Metadata map[string]interface{} `json:"Metadata,omitempty"`
FixedBy string `json:"FixedBy,omitempty"`
FixedIn []Feature `json:"FixedIn,omitempty"`
}

func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
Expand Down Expand Up @@ -115,6 +117,7 @@ func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) {
Description: v.Description,
Link: v.Link,
Severity: severity,
Metadata: v.Metadata,
FixedIn: dbFeatures,
}, nil
}
Expand All @@ -126,6 +129,7 @@ func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability, withFixedIn b
Description: dbVuln.Description,
Link: dbVuln.Link,
Severity: string(dbVuln.Severity),
Metadata: dbVuln.Metadata,
}

if withFixedIn {
Expand Down
1 change: 1 addition & 0 deletions cmd/clair/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
_ "github.com/coreos/clair/updater/fetchers/debian"
_ "github.com/coreos/clair/updater/fetchers/rhel"
_ "github.com/coreos/clair/updater/fetchers/ubuntu"
_ "github.com/coreos/clair/updater/metadata_fetchers/nvd"

_ "github.com/coreos/clair/worker/detectors/data/aci"
_ "github.com/coreos/clair/worker/detectors/data/docker"
Expand Down
19 changes: 19 additions & 0 deletions database/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
package database

import (
"database/sql/driver"
"encoding/json"
"time"

"github.com/coreos/clair/utils/types"
Expand Down Expand Up @@ -65,6 +67,8 @@ type Vulnerability struct {
Link string
Severity types.Priority

Metadata MetadataMap

FixedIn []FeatureVersion
LayersIntroducingVulnerability []Layer

Expand All @@ -73,6 +77,21 @@ type Vulnerability struct {
FixedBy types.Version `json:",omitempty"`
}

type MetadataMap map[string]interface{}

func (mm *MetadataMap) Scan(value interface{}) error {
val, ok := value.([]byte)
if !ok {
return nil
}
return json.Unmarshal(val, mm)
}

func (mm *MetadataMap) Value() (driver.Value, error) {
json, err := json.Marshal(*mm)
return string(json), err
}

type VulnerabilityNotification struct {
Model

Expand Down
2 changes: 1 addition & 1 deletion database/pgsql/layer.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func (pgSQL *pgSQL) loadAffectedBy(featureVersions []database.FeatureVersion) er
var vulnerability database.Vulnerability
err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name,
&vulnerability.Description, &vulnerability.Link, &vulnerability.Severity,
&vulnerability.Namespace.Name, &vulnerability.FixedBy)
&vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy)
if err != nil {
return handleError("s_featureversions_vulnerabilities.Scan()", err)
}
Expand Down
1 change: 1 addition & 0 deletions database/pgsql/migrations/20151222113213_Initial.sql
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ CREATE TABLE IF NOT EXISTS Vulnerability (
description TEXT NULL,
link VARCHAR(128) NULL,
severity severity NOT NULL,
metadata TEXT NULL,

UNIQUE (namespace_id, name));

Expand Down
14 changes: 8 additions & 6 deletions database/pgsql/queries.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ func init() {
ORDER BY ltree.ordering`

queries["s_featureversions_vulnerabilities"] = `
SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, vn.name,
vfif.version
SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata,
vn.name, vfif.version
FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v,
Namespace vn, Vulnerability_FixedIn_Feature vfif
WHERE vafv.featureversion_id = ANY($1::integer[])
Expand Down Expand Up @@ -144,20 +144,22 @@ func init() {

// vulnerability.go
queries["f_vulnerability"] = `
SELECT v.id, n.id, v.description, v.link, v.severity, vfif.version, f.id, f.Name
SELECT v.id, n.id, v.description, v.link, v.severity, v.metadata, vfif.version, f.id, f.Name
FROM Vulnerability v
JOIN Namespace n ON v.namespace_id = n.id
LEFT JOIN Vulnerability_FixedIn_Feature vfif ON v.id = vfif.vulnerability_id
LEFT JOIN Feature f ON vfif.feature_id = f.id
WHERE n.Name = $1 AND v.Name = $2`

queries["i_vulnerability"] = `
INSERT INTO Vulnerability(namespace_id, name, description, link, severity)
VALUES($1, $2, $3, $4, $5)
INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata)
VALUES($1, $2, $3, $4, $5, $6)
RETURNING id`

queries["u_vulnerability"] = `
UPDATE Vulnerability SET description = $2, link = $3, severity = $4 WHERE id = $1`
UPDATE Vulnerability
SET description = $2, link = $3, severity = $4, metadata = $5
WHERE id = $1`

queries["i_vulnerability_fixedin_feature"] = `
INSERT INTO Vulnerability_FixedIn_Feature(vulnerability_id, feature_id, version)
Expand Down
24 changes: 20 additions & 4 deletions database/pgsql/vulnerability.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@ package pgsql

import (
"database/sql"
"encoding/json"
"fmt"
"reflect"
"time"

"github.com/coreos/clair/database"
Expand Down Expand Up @@ -50,8 +52,8 @@ func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vuln
var featureVersionFeatureName zero.String

err := rows.Scan(&vulnerability.ID, &vulnerability.Namespace.ID, &vulnerability.Description,
&vulnerability.Link, &vulnerability.Severity, &featureVersionVersion, &featureVersionID,
&featureVersionFeatureName)
&vulnerability.Link, &vulnerability.Severity, &vulnerability.Metadata,
&featureVersionVersion, &featureVersionID, &featureVersionFeatureName)
if err != nil {
return vulnerability, handleError("f_vulnerability.Scan()", err)
}
Expand Down Expand Up @@ -139,6 +141,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
if vulnerability.Description == existingVulnerability.Description &&
vulnerability.Link == existingVulnerability.Link &&
vulnerability.Severity == existingVulnerability.Severity &&
reflect.DeepEqual(castMetadata(vulnerability.Metadata), existingVulnerability.Metadata) &&
len(newFixedInFeatureVersions) == 0 &&
len(updatedFixedInFeatureVersions) == 0 {

Expand Down Expand Up @@ -191,7 +194,8 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
if existingVulnerability.ID == 0 {
// Insert new vulnerability.
err = tx.QueryRow(getQuery("i_vulnerability"), namespaceID, vulnerability.Name,
vulnerability.Description, vulnerability.Link, &vulnerability.Severity).Scan(&vulnerability.ID)
vulnerability.Description, vulnerability.Link, &vulnerability.Severity,
&vulnerability.Metadata).Scan(&vulnerability.ID)
if err != nil {
tx.Rollback()
return handleError("i_vulnerability", err)
Expand All @@ -202,7 +206,8 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
vulnerability.Link != existingVulnerability.Link ||
vulnerability.Severity != existingVulnerability.Severity {
_, err = tx.Exec(getQuery("u_vulnerability"), existingVulnerability.ID,
vulnerability.Description, vulnerability.Link, &vulnerability.Severity)
vulnerability.Description, vulnerability.Link, &vulnerability.Severity,
&vulnerability.Metadata)
if err != nil {
tx.Rollback()
return handleError("u_vulnerability", err)
Expand Down Expand Up @@ -244,6 +249,17 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er
return nil
}

// castMetadata marshals the given database.MetadataMap and unmarshals it again to make sure that
// everything has the interface{} type.
// It is required when comparing crafted MetadataMap against MetadataMap that we get from the
// database.
func castMetadata(m database.MetadataMap) database.MetadataMap {
c := make(database.MetadataMap)
j, _ := json.Marshal(m)
json.Unmarshal(j, &c)
return c
}

func diffFixedIn(vulnerability, existingVulnerability database.Vulnerability) (newFixedIn, updatedFixedIn []database.FeatureVersion) {
// Build FeatureVersion.Feature.Namespace.Name:FeatureVersion.Feature.Name (NaN) structures.
vulnerabilityFixedInNameMap, vulnerabilityFixedInNameSlice := createFeatureVersionNameMap(vulnerability.FixedIn)
Expand Down
12 changes: 12 additions & 0 deletions database/pgsql/vulnerability_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package pgsql

import (
"reflect"
"testing"

"github.com/coreos/clair/database"
Expand Down Expand Up @@ -195,13 +196,22 @@ func TestInsertVulnerability(t *testing.T) {
}

// Insert a simple vulnerability and find it.
v1meta := make(map[string]interface{})
v1meta["TestInsertVulnerabilityMetadata1"] = "TestInsertVulnerabilityMetadataValue1"
v1meta["TestInsertVulnerabilityMetadata2"] = struct {
Test string
}{
Test: "TestInsertVulnerabilityMetadataValue1",
}

v1 := database.Vulnerability{
Name: "TestInsertVulnerability1",
Namespace: n1,
FixedIn: []database.FeatureVersion{f1, f3, f6, f7},
Severity: types.Low,
Description: "TestInsertVulnerabilityDescription1",
Link: "TestInsertVulnerabilityLink1",
Metadata: v1meta,
}
err = datastore.InsertVulnerabilities([]database.Vulnerability{v1})
if assert.Nil(t, err) {
Expand Down Expand Up @@ -245,6 +255,8 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) {
assert.Equal(t, expected.Description, actual.Description)
assert.Equal(t, expected.Link, actual.Link)
assert.Equal(t, expected.Severity, actual.Severity)
assert.True(t, reflect.DeepEqual(castMetadata(expected.Metadata), actual.Metadata), "Got metadata %#v, expected %#v", actual.Metadata, castMetadata(expected.Metadata))

if assert.Len(t, actual.FixedIn, len(expected.FixedIn)) {
for _, actualFeatureVersion := range actual.FixedIn {
found := false
Expand Down
4 changes: 3 additions & 1 deletion updater/fetchers/ubuntu/ubuntu.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,5 +428,7 @@ func ubuntuPriorityToSeverity(priority string) types.Priority {

// Clean deletes any allocated resources.
func (fetcher *UbuntuFetcher) Clean() {
os.RemoveAll(fetcher.repositoryLocalPath)
if fetcher.repositoryLocalPath != "" {
os.RemoveAll(fetcher.repositoryLocalPath)
}
}
64 changes: 64 additions & 0 deletions updater/metadata_fetchers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package updater

import (
"sync"

"github.com/coreos/clair/database"
)

var metadataFetchers = make(map[string]MetadataFetcher)

type VulnerabilityWithLock struct {
*database.Vulnerability
Lock sync.Mutex
}

// MetadataFetcher
type MetadataFetcher interface {
// Load runs right before the Updater calls AddMetadata for each vulnerabilities.
Load(database.Datastore) error

// AddMetadata adds metadata to the given database.Vulnerability.
// It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map.
AddMetadata(*VulnerabilityWithLock) error

// Unload runs right after the Updater finished calling AddMetadata for every vulnerabilities.
Unload()

// Clean deletes any allocated resources.
// It is invoked when Clair stops.
Clean()
}

// RegisterFetcher makes a Fetcher available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func RegisterMetadataFetcher(name string, f MetadataFetcher) {
if name == "" {
panic("updater: could not register a MetadataFetcher with an empty name")
}

if f == nil {
panic("updater: could not register a nil MetadataFetcher")
}

if _, dup := fetchers[name]; dup {
panic("updater: RegisterMetadataFetcher called twice for " + name)
}

metadataFetchers[name] = f
}
19 changes: 19 additions & 0 deletions updater/metadata_fetchers/nvd/nested_read_closer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package nvd

import "io"

// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed
// io.ReadCloser.
// It allows chaining io.ReadCloser together and still keep the ability to close them all in a
// simple manner.
type NestedReadCloser struct {
io.Reader
NestedReadClosers []io.ReadCloser
}

// Close closes the gzip.Reader and the underlying io.ReadCloser.
func (nrc *NestedReadCloser) Close() {
for _, nestedReadCloser := range nrc.NestedReadClosers {
nestedReadCloser.Close()
}
}
Loading

0 comments on commit 5fdd9d1

Please sign in to comment.