From 5fdd9d1a07220ede12a7009b54641103fcfe2c24 Mon Sep 17 00:00:00 2001 From: Quentin Machu Date: Mon, 1 Feb 2016 18:41:40 -0500 Subject: [PATCH] *: add metadata support along with NVD CVSS --- api/v1/models.go | 18 +- cmd/clair/main.go | 1 + database/models.go | 19 ++ database/pgsql/layer.go | 2 +- .../migrations/20151222113213_Initial.sql | 1 + database/pgsql/queries.go | 14 +- database/pgsql/vulnerability.go | 24 +- database/pgsql/vulnerability_test.go | 12 + updater/fetchers/ubuntu/ubuntu.go | 4 +- updater/metadata_fetchers.go | 64 +++++ .../nvd/nested_read_closer.go | 19 ++ updater/metadata_fetchers/nvd/nvd.go | 228 ++++++++++++++++++ updater/metadata_fetchers/nvd/xml.go | 82 +++++++ updater/updater.go | 56 ++++- utils/string.go | 14 +- utils/types/cvss.go | 225 ----------------- utils/utils_test.go | 3 - 17 files changed, 522 insertions(+), 264 deletions(-) create mode 100644 updater/metadata_fetchers.go create mode 100644 updater/metadata_fetchers/nvd/nested_read_closer.go create mode 100644 updater/metadata_fetchers/nvd/nvd.go create mode 100644 updater/metadata_fetchers/nvd/xml.go delete mode 100644 utils/types/cvss.go diff --git a/api/v1/models.go b/api/v1/models.go index 00d1c88141..e3fd25fa50 100644 --- a/api/v1/models.go +++ b/api/v1/models.go @@ -63,6 +63,7 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil Namespace: dbVuln.Namespace.Name, Description: dbVuln.Description, Severity: string(dbVuln.Severity), + Metadata: dbVuln.Metadata, } if dbVuln.FixedBy != types.MaxVersion { @@ -78,13 +79,14 @@ func LayerFromDatabaseModel(dbLayer database.Layer, withFeatures, withVulnerabil } type Vulnerability struct { - Name string `json:"Name,omitempty"` - Namespace string `json:"Namespace,omitempty"` - Description string `json:"Description,omitempty"` - Link string `json:"Link,omitempty"` - Severity string `json:"Severity,omitempty"` - FixedBy string `json:"FixedBy,omitempty"` - FixedIn []Feature `json:"FixedIn,omitempty"` + Name string `json:"Name,omitempty"` + Namespace string `json:"Namespace,omitempty"` + Description string `json:"Description,omitempty"` + Link string `json:"Link,omitempty"` + Severity string `json:"Severity,omitempty"` + Metadata map[string]interface{} `json:"Metadata,omitempty"` + FixedBy string `json:"FixedBy,omitempty"` + FixedIn []Feature `json:"FixedIn,omitempty"` } func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) { @@ -115,6 +117,7 @@ func (v Vulnerability) DatabaseModel() (database.Vulnerability, error) { Description: v.Description, Link: v.Link, Severity: severity, + Metadata: v.Metadata, FixedIn: dbFeatures, }, nil } @@ -126,6 +129,7 @@ func VulnerabilityFromDatabaseModel(dbVuln database.Vulnerability, withFixedIn b Description: dbVuln.Description, Link: dbVuln.Link, Severity: string(dbVuln.Severity), + Metadata: dbVuln.Metadata, } if withFixedIn { diff --git a/cmd/clair/main.go b/cmd/clair/main.go index 6f5c9fbb74..8b129bb762 100644 --- a/cmd/clair/main.go +++ b/cmd/clair/main.go @@ -31,6 +31,7 @@ import ( _ "github.com/coreos/clair/updater/fetchers/debian" _ "github.com/coreos/clair/updater/fetchers/rhel" _ "github.com/coreos/clair/updater/fetchers/ubuntu" + _ "github.com/coreos/clair/updater/metadata_fetchers/nvd" _ "github.com/coreos/clair/worker/detectors/data/aci" _ "github.com/coreos/clair/worker/detectors/data/docker" diff --git a/database/models.go b/database/models.go index 5202ef54aa..de1e73c6d8 100644 --- a/database/models.go +++ b/database/models.go @@ -15,6 +15,8 @@ package database import ( + "database/sql/driver" + "encoding/json" "time" "github.com/coreos/clair/utils/types" @@ -65,6 +67,8 @@ type Vulnerability struct { Link string Severity types.Priority + Metadata MetadataMap + FixedIn []FeatureVersion LayersIntroducingVulnerability []Layer @@ -73,6 +77,21 @@ type Vulnerability struct { FixedBy types.Version `json:",omitempty"` } +type MetadataMap map[string]interface{} + +func (mm *MetadataMap) Scan(value interface{}) error { + val, ok := value.([]byte) + if !ok { + return nil + } + return json.Unmarshal(val, mm) +} + +func (mm *MetadataMap) Value() (driver.Value, error) { + json, err := json.Marshal(*mm) + return string(json), err +} + type VulnerabilityNotification struct { Model diff --git a/database/pgsql/layer.go b/database/pgsql/layer.go index a0d46773db..cf0200bac9 100644 --- a/database/pgsql/layer.go +++ b/database/pgsql/layer.go @@ -163,7 +163,7 @@ func (pgSQL *pgSQL) loadAffectedBy(featureVersions []database.FeatureVersion) er var vulnerability database.Vulnerability err := rows.Scan(&featureversionID, &vulnerability.ID, &vulnerability.Name, &vulnerability.Description, &vulnerability.Link, &vulnerability.Severity, - &vulnerability.Namespace.Name, &vulnerability.FixedBy) + &vulnerability.Metadata, &vulnerability.Namespace.Name, &vulnerability.FixedBy) if err != nil { return handleError("s_featureversions_vulnerabilities.Scan()", err) } diff --git a/database/pgsql/migrations/20151222113213_Initial.sql b/database/pgsql/migrations/20151222113213_Initial.sql index 53b3b5576b..d1db537b6b 100644 --- a/database/pgsql/migrations/20151222113213_Initial.sql +++ b/database/pgsql/migrations/20151222113213_Initial.sql @@ -88,6 +88,7 @@ CREATE TABLE IF NOT EXISTS Vulnerability ( description TEXT NULL, link VARCHAR(128) NULL, severity severity NOT NULL, + metadata TEXT NULL, UNIQUE (namespace_id, name)); diff --git a/database/pgsql/queries.go b/database/pgsql/queries.go index e255950db4..bd5f0db19a 100644 --- a/database/pgsql/queries.go +++ b/database/pgsql/queries.go @@ -104,8 +104,8 @@ func init() { ORDER BY ltree.ordering` queries["s_featureversions_vulnerabilities"] = ` - SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, vn.name, - vfif.version + SELECT vafv.featureversion_id, v.id, v.name, v.description, v.link, v.severity, v.metadata, + vn.name, vfif.version FROM Vulnerability_Affects_FeatureVersion vafv, Vulnerability v, Namespace vn, Vulnerability_FixedIn_Feature vfif WHERE vafv.featureversion_id = ANY($1::integer[]) @@ -144,7 +144,7 @@ func init() { // vulnerability.go queries["f_vulnerability"] = ` - SELECT v.id, n.id, v.description, v.link, v.severity, vfif.version, f.id, f.Name + SELECT v.id, n.id, v.description, v.link, v.severity, v.metadata, vfif.version, f.id, f.Name FROM Vulnerability v JOIN Namespace n ON v.namespace_id = n.id LEFT JOIN Vulnerability_FixedIn_Feature vfif ON v.id = vfif.vulnerability_id @@ -152,12 +152,14 @@ func init() { WHERE n.Name = $1 AND v.Name = $2` queries["i_vulnerability"] = ` - INSERT INTO Vulnerability(namespace_id, name, description, link, severity) - VALUES($1, $2, $3, $4, $5) + INSERT INTO Vulnerability(namespace_id, name, description, link, severity, metadata) + VALUES($1, $2, $3, $4, $5, $6) RETURNING id` queries["u_vulnerability"] = ` - UPDATE Vulnerability SET description = $2, link = $3, severity = $4 WHERE id = $1` + UPDATE Vulnerability + SET description = $2, link = $3, severity = $4, metadata = $5 + WHERE id = $1` queries["i_vulnerability_fixedin_feature"] = ` INSERT INTO Vulnerability_FixedIn_Feature(vulnerability_id, feature_id, version) diff --git a/database/pgsql/vulnerability.go b/database/pgsql/vulnerability.go index a63d0de741..4035e258be 100644 --- a/database/pgsql/vulnerability.go +++ b/database/pgsql/vulnerability.go @@ -16,7 +16,9 @@ package pgsql import ( "database/sql" + "encoding/json" "fmt" + "reflect" "time" "github.com/coreos/clair/database" @@ -50,8 +52,8 @@ func (pgSQL *pgSQL) FindVulnerability(namespaceName, name string) (database.Vuln var featureVersionFeatureName zero.String err := rows.Scan(&vulnerability.ID, &vulnerability.Namespace.ID, &vulnerability.Description, - &vulnerability.Link, &vulnerability.Severity, &featureVersionVersion, &featureVersionID, - &featureVersionFeatureName) + &vulnerability.Link, &vulnerability.Severity, &vulnerability.Metadata, + &featureVersionVersion, &featureVersionID, &featureVersionFeatureName) if err != nil { return vulnerability, handleError("f_vulnerability.Scan()", err) } @@ -139,6 +141,7 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er if vulnerability.Description == existingVulnerability.Description && vulnerability.Link == existingVulnerability.Link && vulnerability.Severity == existingVulnerability.Severity && + reflect.DeepEqual(castMetadata(vulnerability.Metadata), existingVulnerability.Metadata) && len(newFixedInFeatureVersions) == 0 && len(updatedFixedInFeatureVersions) == 0 { @@ -191,7 +194,8 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er if existingVulnerability.ID == 0 { // Insert new vulnerability. err = tx.QueryRow(getQuery("i_vulnerability"), namespaceID, vulnerability.Name, - vulnerability.Description, vulnerability.Link, &vulnerability.Severity).Scan(&vulnerability.ID) + vulnerability.Description, vulnerability.Link, &vulnerability.Severity, + &vulnerability.Metadata).Scan(&vulnerability.ID) if err != nil { tx.Rollback() return handleError("i_vulnerability", err) @@ -202,7 +206,8 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er vulnerability.Link != existingVulnerability.Link || vulnerability.Severity != existingVulnerability.Severity { _, err = tx.Exec(getQuery("u_vulnerability"), existingVulnerability.ID, - vulnerability.Description, vulnerability.Link, &vulnerability.Severity) + vulnerability.Description, vulnerability.Link, &vulnerability.Severity, + &vulnerability.Metadata) if err != nil { tx.Rollback() return handleError("u_vulnerability", err) @@ -244,6 +249,17 @@ func (pgSQL *pgSQL) insertVulnerability(vulnerability database.Vulnerability) er return nil } +// castMetadata marshals the given database.MetadataMap and unmarshals it again to make sure that +// everything has the interface{} type. +// It is required when comparing crafted MetadataMap against MetadataMap that we get from the +// database. +func castMetadata(m database.MetadataMap) database.MetadataMap { + c := make(database.MetadataMap) + j, _ := json.Marshal(m) + json.Unmarshal(j, &c) + return c +} + func diffFixedIn(vulnerability, existingVulnerability database.Vulnerability) (newFixedIn, updatedFixedIn []database.FeatureVersion) { // Build FeatureVersion.Feature.Namespace.Name:FeatureVersion.Feature.Name (NaN) structures. vulnerabilityFixedInNameMap, vulnerabilityFixedInNameSlice := createFeatureVersionNameMap(vulnerability.FixedIn) diff --git a/database/pgsql/vulnerability_test.go b/database/pgsql/vulnerability_test.go index fbb266ff54..6db3b08b9b 100644 --- a/database/pgsql/vulnerability_test.go +++ b/database/pgsql/vulnerability_test.go @@ -15,6 +15,7 @@ package pgsql import ( + "reflect" "testing" "github.com/coreos/clair/database" @@ -195,6 +196,14 @@ func TestInsertVulnerability(t *testing.T) { } // Insert a simple vulnerability and find it. + v1meta := make(map[string]interface{}) + v1meta["TestInsertVulnerabilityMetadata1"] = "TestInsertVulnerabilityMetadataValue1" + v1meta["TestInsertVulnerabilityMetadata2"] = struct { + Test string + }{ + Test: "TestInsertVulnerabilityMetadataValue1", + } + v1 := database.Vulnerability{ Name: "TestInsertVulnerability1", Namespace: n1, @@ -202,6 +211,7 @@ func TestInsertVulnerability(t *testing.T) { Severity: types.Low, Description: "TestInsertVulnerabilityDescription1", Link: "TestInsertVulnerabilityLink1", + Metadata: v1meta, } err = datastore.InsertVulnerabilities([]database.Vulnerability{v1}) if assert.Nil(t, err) { @@ -245,6 +255,8 @@ func equalsVuln(t *testing.T, expected, actual *database.Vulnerability) { assert.Equal(t, expected.Description, actual.Description) assert.Equal(t, expected.Link, actual.Link) assert.Equal(t, expected.Severity, actual.Severity) + assert.True(t, reflect.DeepEqual(castMetadata(expected.Metadata), actual.Metadata), "Got metadata %#v, expected %#v", actual.Metadata, castMetadata(expected.Metadata)) + if assert.Len(t, actual.FixedIn, len(expected.FixedIn)) { for _, actualFeatureVersion := range actual.FixedIn { found := false diff --git a/updater/fetchers/ubuntu/ubuntu.go b/updater/fetchers/ubuntu/ubuntu.go index 776401e3e7..ad81ccb581 100644 --- a/updater/fetchers/ubuntu/ubuntu.go +++ b/updater/fetchers/ubuntu/ubuntu.go @@ -428,5 +428,7 @@ func ubuntuPriorityToSeverity(priority string) types.Priority { // Clean deletes any allocated resources. func (fetcher *UbuntuFetcher) Clean() { - os.RemoveAll(fetcher.repositoryLocalPath) + if fetcher.repositoryLocalPath != "" { + os.RemoveAll(fetcher.repositoryLocalPath) + } } diff --git a/updater/metadata_fetchers.go b/updater/metadata_fetchers.go new file mode 100644 index 0000000000..72e764af02 --- /dev/null +++ b/updater/metadata_fetchers.go @@ -0,0 +1,64 @@ +// Copyright 2015 clair authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package updater + +import ( + "sync" + + "github.com/coreos/clair/database" +) + +var metadataFetchers = make(map[string]MetadataFetcher) + +type VulnerabilityWithLock struct { + *database.Vulnerability + Lock sync.Mutex +} + +// MetadataFetcher +type MetadataFetcher interface { + // Load runs right before the Updater calls AddMetadata for each vulnerabilities. + Load(database.Datastore) error + + // AddMetadata adds metadata to the given database.Vulnerability. + // It is expected that the fetcher uses .Lock.Lock() when manipulating the Metadata map. + AddMetadata(*VulnerabilityWithLock) error + + // Unload runs right after the Updater finished calling AddMetadata for every vulnerabilities. + Unload() + + // Clean deletes any allocated resources. + // It is invoked when Clair stops. + Clean() +} + +// RegisterFetcher makes a Fetcher available by the provided name. +// If Register is called twice with the same name or if driver is nil, +// it panics. +func RegisterMetadataFetcher(name string, f MetadataFetcher) { + if name == "" { + panic("updater: could not register a MetadataFetcher with an empty name") + } + + if f == nil { + panic("updater: could not register a nil MetadataFetcher") + } + + if _, dup := fetchers[name]; dup { + panic("updater: RegisterMetadataFetcher called twice for " + name) + } + + metadataFetchers[name] = f +} diff --git a/updater/metadata_fetchers/nvd/nested_read_closer.go b/updater/metadata_fetchers/nvd/nested_read_closer.go new file mode 100644 index 0000000000..3a99b174c6 --- /dev/null +++ b/updater/metadata_fetchers/nvd/nested_read_closer.go @@ -0,0 +1,19 @@ +package nvd + +import "io" + +// NestedReadCloser wraps an io.Reader and implements io.ReadCloser by closing every embed +// io.ReadCloser. +// It allows chaining io.ReadCloser together and still keep the ability to close them all in a +// simple manner. +type NestedReadCloser struct { + io.Reader + NestedReadClosers []io.ReadCloser +} + +// Close closes the gzip.Reader and the underlying io.ReadCloser. +func (nrc *NestedReadCloser) Close() { + for _, nestedReadCloser := range nrc.NestedReadClosers { + nestedReadCloser.Close() + } +} diff --git a/updater/metadata_fetchers/nvd/nvd.go b/updater/metadata_fetchers/nvd/nvd.go new file mode 100644 index 0000000000..4f57bb6200 --- /dev/null +++ b/updater/metadata_fetchers/nvd/nvd.go @@ -0,0 +1,228 @@ +package nvd + +import ( + "bufio" + "compress/gzip" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/coreos/clair/database" + "github.com/coreos/clair/updater" + cerrors "github.com/coreos/clair/utils/errors" + "github.com/coreos/pkg/capnslog" +) + +const ( + dataFeedURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz" + dataFeedMetaURL string = "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.meta" + + metadataKey string = "NVD" +) + +var ( + log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/metadata_fetchers") +) + +type NVDMetadataFetcher struct { + localPath string + dataFeedHashes map[string]string + lock sync.Mutex + + metadata map[string]NVDMetadata +} + +type NVDMetadata struct { + CVSSv2 NVDmetadataCVSSv2 +} + +type NVDmetadataCVSSv2 struct { + Vectors string + Score float64 +} + +func init() { + updater.RegisterMetadataFetcher("NVD", &NVDMetadataFetcher{}) +} + +func (fetcher *NVDMetadataFetcher) Load(datastore database.Datastore) error { + fetcher.lock.Lock() + defer fetcher.lock.Unlock() + + var err error + fetcher.metadata = make(map[string]NVDMetadata) + + // Init if necessary. + if fetcher.localPath == "" { + // Create a temporary folder to store the NVD data and create hashes struct. + if fetcher.localPath, err = ioutil.TempDir(os.TempDir(), "nvd-data"); err != nil { + return cerrors.ErrFilesystem + } + + fetcher.dataFeedHashes = make(map[string]string) + } + + // Get data feeds. + dataFeedReaders, dataFeedHashes, err := getDataFeeds(fetcher.dataFeedHashes, fetcher.localPath) + if err != nil { + return err + } + fetcher.dataFeedHashes = dataFeedHashes + + // Parse data feeds. + for dataFeedName, dataFeedReader := range dataFeedReaders { + var nvd nvd + if err = xml.NewDecoder(dataFeedReader).Decode(&nvd); err != nil { + log.Errorf("could not decode NVD data feed '%s': %s", dataFeedName, err) + return cerrors.ErrCouldNotParse + } + + // For each entry of this data feed: + for _, nvdEntry := range nvd.Entries { + // Create metadata entry. + if metadata := nvdEntry.Metadata(); metadata != nil { + fetcher.metadata[nvdEntry.Name] = *metadata + } + } + + dataFeedReader.Close() + } + + return nil +} + +func (fetcher *NVDMetadataFetcher) AddMetadata(vulnerability *updater.VulnerabilityWithLock) error { + fetcher.lock.Lock() + defer fetcher.lock.Unlock() + + if nvdMetadata, ok := fetcher.metadata[vulnerability.Name]; ok { + vulnerability.Lock.Lock() + defer vulnerability.Lock.Unlock() + + // Create Metadata map if necessary. + if vulnerability.Metadata == nil { + vulnerability.Metadata = make(map[string]interface{}) + } + + vulnerability.Metadata[metadataKey] = nvdMetadata + } + + return nil +} + +func (fetcher *NVDMetadataFetcher) Unload() { + fetcher.lock.Lock() + defer fetcher.lock.Unlock() + + fetcher.metadata = nil +} + +func (fetcher *NVDMetadataFetcher) Clean() { + fetcher.lock.Lock() + defer fetcher.lock.Unlock() + + if fetcher.localPath != "" { + os.RemoveAll(fetcher.localPath) + } +} + +func getDataFeeds(dataFeedHashes map[string]string, localPath string) (map[string]NestedReadCloser, map[string]string, error) { + var dataFeedNames []string + for y := 2002; y <= time.Now().Year(); y++ { + dataFeedNames = append(dataFeedNames, strconv.Itoa(y)) + } + + // Get hashes for these feeds. + for _, dataFeedName := range dataFeedNames { + hash, err := getHashFromMetaURL(fmt.Sprintf(dataFeedMetaURL, dataFeedName)) + if err != nil { + log.Warningf("could get get NVD data feed hash '%s': %s", dataFeedName, err) + + // It's not a big deal, no need interrupt, we're just going to download it again then. + continue + } + + dataFeedHashes[dataFeedName] = hash + } + + // Create io.Reader for every data feed. + dataFeedReaders := make(map[string]NestedReadCloser) + for _, dataFeedName := range dataFeedNames { + fileName := localPath + dataFeedName + ".xml" + + if h, ok := dataFeedHashes[dataFeedName]; ok && h == dataFeedHashes[dataFeedName] { + // The hash is known, the disk should contains the feed. Try to read from it. + if localPath != "" { + if f, err := os.Open(fileName); err == nil { + dataFeedReaders[dataFeedName] = NestedReadCloser{ + Reader: f, + NestedReadClosers: []io.ReadCloser{f}, + } + continue + } + } + + // Download data feed. + r, err := http.Get(fmt.Sprintf(dataFeedURL, dataFeedName)) + if err != nil { + log.Errorf("could not download NVD data feed file '%s': %s", dataFeedName, err) + return dataFeedReaders, dataFeedHashes, cerrors.ErrCouldNotDownload + } + + // Un-gzip it. + gr, err := gzip.NewReader(r.Body) + if err != nil { + log.Errorf("could not read NVD data feed file '%s': %s", dataFeedName, err) + return dataFeedReaders, dataFeedHashes, cerrors.ErrCouldNotDownload + } + + // Store it to a file at the same time if possible. + if f, err := os.Create(fileName); err == nil { + nrc := NestedReadCloser{ + Reader: io.TeeReader(gr, f), + NestedReadClosers: []io.ReadCloser{r.Body, gr, f}, + } + dataFeedReaders[dataFeedName] = nrc + } else { + nrc := NestedReadCloser{ + Reader: gr, + NestedReadClosers: []io.ReadCloser{gr, r.Body}, + } + dataFeedReaders[dataFeedName] = nrc + + log.Warningf("could not store NVD data feed to filesystem: %s", err) + } + } + } + + return dataFeedReaders, dataFeedHashes, nil +} + +func getHashFromMetaURL(metaURL string) (string, error) { + r, err := http.Get(metaURL) + if err != nil { + return "", err + } + defer r.Body.Close() + + scanner := bufio.NewScanner(r.Body) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "sha256:") { + return strings.TrimPrefix(line, "sha256:"), nil + } + } + if err := scanner.Err(); err != nil { + return "", err + } + + return "", errors.New("invalid .meta file format") +} diff --git a/updater/metadata_fetchers/nvd/xml.go b/updater/metadata_fetchers/nvd/xml.go new file mode 100644 index 0000000000..db13c897d7 --- /dev/null +++ b/updater/metadata_fetchers/nvd/xml.go @@ -0,0 +1,82 @@ +package nvd + +import ( + "fmt" + "strings" +) + +type nvd struct { + Entries []nvdEntry `xml:"entry"` +} + +type nvdEntry struct { + Name string `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cve-id"` + CVSS nvdCVSS `xml:"http://scap.nist.gov/schema/vulnerability/0.4 cvss"` +} + +type nvdCVSS struct { + BaseMetrics nvdCVSSBaseMetrics `xml:"http://scap.nist.gov/schema/cvss-v2/0.2 base_metrics"` +} + +type nvdCVSSBaseMetrics struct { + Score float64 `xml:"score"` + AccessVector string `xml:"access-vector"` + AccessComplexity string `xml:"access-complexity"` + Authentication string `xml:"authentication"` + ConfImpact string `xml:"confidentiality-impact"` + IntegImpact string `xml:"integrity-impact"` + AvailImpact string `xml:"avaibility-impact"` +} + +var vectorValuesToLetters map[string]string + +func init() { + vectorValuesToLetters = make(map[string]string) + vectorValuesToLetters["NETWORK"] = "N" + vectorValuesToLetters["ADJACENT_NETWORK"] = "A" + vectorValuesToLetters["LOCAL"] = "L" + vectorValuesToLetters["HIGH"] = "H" + vectorValuesToLetters["MEDIUM"] = "M" + vectorValuesToLetters["LOW"] = "L" + vectorValuesToLetters["NONE"] = "N" + vectorValuesToLetters["SINGLE_INSTANCE"] = "S" + vectorValuesToLetters["MULTIPLE_INSTANCES"] = "M" + vectorValuesToLetters["PARTIAL"] = "P" + vectorValuesToLetters["COMPLETE"] = "C" +} + +func (n nvdEntry) Metadata() *NVDMetadata { + metadata := &NVDMetadata{ + CVSSv2: NVDmetadataCVSSv2{ + Vectors: n.CVSS.BaseMetrics.String(), + Score: n.CVSS.BaseMetrics.Score, + }, + } + + if metadata.CVSSv2.Vectors == "" { + return nil + } + return metadata +} + +func (n nvdCVSSBaseMetrics) String() string { + var str string + addVec(&str, "AV", n.AccessVector) + addVec(&str, "AC", n.AccessComplexity) + addVec(&str, "Au", n.Authentication) + addVec(&str, "C", n.ConfImpact) + addVec(&str, "I", n.IntegImpact) + addVec(&str, "A", n.AvailImpact) + str = strings.TrimSuffix(str, "/") + return str +} + +func addVec(str *string, vec, val string) { + if val != "" { + if let, ok := vectorValuesToLetters[val]; ok { + *str = fmt.Sprintf("%s%s:%s/", *str, vec, let) + } else { + log.Warningf("unknown value '%v' for CVSSv2 vector '%s'", val, vec) + } + } +} diff --git a/updater/updater.go b/updater/updater.go index 1f3f54df89..d2b7946073 100644 --- a/updater/updater.go +++ b/updater/updater.go @@ -19,6 +19,7 @@ package updater import ( "math/rand" "strconv" + "sync" "time" "github.com/coreos/clair/config" @@ -144,6 +145,9 @@ func Run(config *config.UpdaterConfig, datastore database.Datastore, st *utils.S } // Clean resources. + for _, metadataFetcher := range metadataFetchers { + metadataFetcher.Clean() + } for _, fetcher := range fetchers { fetcher.Clean() } @@ -161,10 +165,8 @@ func Update(datastore database.Datastore) { // Fetch updates. status, vulnerabilities, flags, notes := fetch(datastore) - // TODO(Quentin-M): Complete informations using NVD - // Insert vulnerabilities. - log.Tracef("beginning insertion of %d vulnerabilities for update", len(vulnerabilities)) + log.Tracef("inserting %d vulnerabilities for update", len(vulnerabilities)) err := datastore.InsertVulnerabilities(vulnerabilities) if err != nil { promUpdaterErrorsTotal.Inc() @@ -204,6 +206,7 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st flags := make(map[string]string) // Fetch updates in parallel. + log.Info("fetching vulnerability updates") var responseC = make(chan *FetcherResponse, 0) for n, f := range fetchers { go func(name string, fetcher Fetcher) { @@ -233,7 +236,52 @@ func fetch(datastore database.Datastore) (bool, []database.Vulnerability, map[st } close(responseC) - return status, vulnerabilities, flags, notes + return status, addMetadata(datastore, vulnerabilities), flags, notes +} + +// Add metadata to the specified vulnerabilities using the registered MetadataFetchers, in parallel. +func addMetadata(datastore database.Datastore, vulnerabilities []database.Vulnerability) []database.Vulnerability { + if len(metadataFetchers) == 0 { + return vulnerabilities + } + + log.Info("adding metadata to vulnerabilities") + + // Wrap vulnerabilities in VulnerabilityWithLock. + // It ensures that only one metadata fetcher at a time can modify the Metadata map. + vulnerabilitiesWithLocks := make([]*VulnerabilityWithLock, 0, len(vulnerabilities)) + for i := 0; i < len(vulnerabilities); i++ { + vulnerabilitiesWithLocks = append(vulnerabilitiesWithLocks, &VulnerabilityWithLock{ + Vulnerability: &vulnerabilities[i], + }) + } + + var wg sync.WaitGroup + wg.Add(len(metadataFetchers)) + + for n, f := range metadataFetchers { + go func(name string, metadataFetcher MetadataFetcher) { + defer wg.Done() + + // Load the metadata fetcher. + if err := metadataFetcher.Load(datastore); err != nil { + promUpdaterErrorsTotal.Inc() + log.Errorf("an error occured when loading metadata fetcher '%s': %s.", name, err) + return + } + + // Add metadata to each vulnerability. + for _, vulnerability := range vulnerabilitiesWithLocks { + metadataFetcher.AddMetadata(vulnerability) + } + + metadataFetcher.Unload() + }(n, f) + } + + wg.Wait() + + return vulnerabilities } func getLastUpdate(datastore database.Datastore) time.Time { diff --git a/utils/string.go b/utils/string.go index a366c2f549..78f7b7a9d7 100644 --- a/utils/string.go +++ b/utils/string.go @@ -14,22 +14,10 @@ package utils -import ( - "crypto/sha1" - "encoding/hex" - "regexp" -) +import "regexp" var urlParametersRegexp = regexp.MustCompile(`(\?|\&)([^=]+)\=([^ &]+)`) -// Hash returns an unique hash of the given string. -func Hash(str string) string { - h := sha1.New() - h.Write([]byte(str)) - bs := h.Sum(nil) - return hex.EncodeToString(bs) -} - // CleanURL removes all parameters from an URL. func CleanURL(str string) string { return urlParametersRegexp.ReplaceAllString(str, "") diff --git a/utils/types/cvss.go b/utils/types/cvss.go deleted file mode 100644 index 3c00b81dba..0000000000 --- a/utils/types/cvss.go +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright 2015 clair authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package types - -// -// import "fmt" -// -// // CVSSv2 represents the Common Vulnerability Scoring System (CVSS), that assesses the severity of -// // vulnerabilities. -// // It describes the CVSS score, but also a vector describing the components from which the score -// // was calculated. This provides users of the score confidence in its correctness and provides -// // insight into the nature of the vulnerability. -// // -// // Reference: https://nvd.nist.gov/CVSS/Vector-v2.aspx -// type CVSSv2 struct { -// // Base Vectors -// AccessVector CVSSValue -// AccessComplexity CVSSValue -// Authentication CVSSValue -// ConfImpact CVSSValue -// IntegImpact CVSSValue -// AvailImpact CVSSValue -// // Temporal Vectors -// Exploitability CVSSValue -// RemediationLevel CVSSValue -// ReportConfidence CVSSValue -// // Environmental Vectors -// CollateralDamagePotential CVSSValue -// TargetDistribution CVSSValue -// SystemConfidentialityRequirement CVSSValue -// SystemIntegrityRequirement CVSSValue -// SystemAvailabilityRequirement CVSSValue -// } -// -// func NewCVSSv2(value string) (*CVSSv2, error) { -// -// } -// -// // CVSSValue is the comprehensible value for a CVSS metric. -// type CVSSValue string -// -// // Metric acronym + Value abbreviation -> Comprehensible metric value. -// var toValue map[string]func(string) (CVSSValue, error) -// -// func init() { -// parsers = make(map[string]func(string) (CVSSValue, error), 14) -// toValue["AV"] = av -// toValue["AC"] = ac -// toValue["Au"] = au -// toValue["C"] = cAndIAndA -// toValue["I"] = cAndIAndA -// toValue["A"] = cAndIAndA -// toValue["E"] = e -// toValue["RL"] = rl -// toValue["RC"] = rc -// toValue["CDP"] = cdp -// toValue["TD"] = td -// toValue["CR"] = crAndIrAndAr -// toValue["IR"] = crAndIrAndAr -// toValue["AR"] = crAndIrAndAr -// } -// -// func av(v string) (CVSSValue, error) { -// switch v { -// case "L": -// return CVSSValue("Local access"), nil -// case "A": -// return CVSSValue("Adjacent Network"), nil -// case "N": -// return CVSSValue("Network"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for AV", v) -// } -// } -// -// func ac(v string) (CVSSValue, error) { -// switch v { -// case "H": -// return CVSSValue("High"), nil -// case "M": -// return CVSSValue("Medium"), nil -// case "L": -// return CVSSValue("Low"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for AC", v) -// } -// } -// -// func au(v string) (CVSSValue, error) { -// switch v { -// case "N": -// return CVSSValue("None required"), nil -// case "S": -// return CVSSValue("Requires single instance"), nil -// case "M": -// return CVSSValue("Requires multiple instances"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for Au", v) -// } -// } -// -// func cAndIAndA(v string) (CVSSValue, error) { -// switch v { -// case "N": -// return CVSSValue("None"), nil -// case "P": -// return CVSSValue("Partial"), nil -// case "C": -// return CVSSValue("Complete"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for C/I/A", v) -// } -// } -// -// func e(v string) (CVSSValue, error) { -// switch v { -// case "U": -// return CVSSValue("Unproven"), nil -// case "POC": -// return CVSSValue("Proof-of-concept"), nil -// case "F": -// return CVSSValue("Functional"), nil -// case "H": -// return CVSSValue("High"), nil -// case "ND": -// return CVSSValue("Not Defined"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for E", v) -// } -// } -// -// func rl(v string) (CVSSValue, error) { -// switch v { -// case "OF": -// return CVSSValue("Official-fix"), nil -// case "T": -// return CVSSValue("Temporary-fix"), nil -// case "W": -// return CVSSValue("Workaround"), nil -// case "U": -// return CVSSValue("Unavailable"), nil -// case "ND": -// return CVSSValue("Not Defined"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for RL", v) -// } -// } -// -// func rc(v string) (CVSSValue, error) { -// switch v { -// case "UC": -// return CVSSValue("Unconfirmed"), nil -// case "UR": -// return CVSSValue("Uncorroborated"), nil -// case "C": -// return CVSSValue("Confirmed"), nil -// case "ND": -// return CVSSValue("Not Defined"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for RC", v) -// } -// } -// -// func cdp(v string) (CVSSValue, error) { -// switch v { -// case "N": -// return CVSSValue("None"), nil -// case "L": -// return CVSSValue("Low"), nil -// case "LM": -// return CVSSValue("Low-Medium"), nil -// case "MH": -// return CVSSValue("Medium-High"), nil -// case "H": -// return CVSSValue("High"), nil -// case "ND": -// return CVSSValue("Not Defined"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for CDP", v) -// } -// } -// -// func td(v string) (CVSSValue, error) { -// switch v { -// case "N": -// return CVSSValue("None (0%)"), nil -// case "L": -// return CVSSValue("Low (1-25%)"), nil -// case "M": -// return CVSSValue("Medium (26-75%)"), nil -// case "H": -// return CVSSValue("High (76-100%)"), nil -// case "ND": -// return CVSSValue("Not Defined"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for TD", v) -// } -// } -// -// func crAndIrAndAr(v string) (CVSSValue, error) { -// switch v { -// case "L": -// return CVSSValue("Low"), nil -// case "M": -// return CVSSValue("Medium"), nil -// case "H": -// return CVSSValue("High"), nil -// case "ND": -// return CVSSValue("Not Defined"), nil -// default: -// return "", fmt.Errorf("%v is not a valid value for CR/IR/AR", v) -// } -// } diff --git a/utils/utils_test.go b/utils/utils_test.go index 8962c84f24..bb4b0ede8b 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -56,9 +56,6 @@ func TestExec(t *testing.T) { // TestString tests the string.go file func TestString(t *testing.T) { - assert.Equal(t, Hash("abc123"), Hash("abc123")) - assert.NotEqual(t, Hash("abc123."), Hash("abc123")) - assert.False(t, Contains("", []string{})) assert.True(t, Contains("a", []string{"a", "b"})) assert.False(t, Contains("c", []string{"a", "b"}))