Implement multiple advisory databases (#244)

Instead of using and creating a single `Database` instance,
you will now create a `DatabaseCollection`, which mirrors the API
of a single `Database`, but queries the results from multiple databases.

src/advisories/cfg.rs

@@ -16,8 +16,14 @@ fn yanked() -> Spanned<LintLevel> {
 pub struct Config {
     /// Path to the local copy of advisory database's git repo (default: ~/.cargo/advisory-db)
     pub db_path: Option<PathBuf>,
+    /// List of paths to local copies of different advisory databases.
+    #[serde(default)]
+    pub db_paths: Vec<PathBuf>,
     /// URL to the advisory database's git repo (default: https://github.com/RustSec/advisory-db)
     pub db_url: Option<String>,
+    /// List of urls to git repositories of different advisory databases.
+    #[serde(default)]
+    pub db_urls: Vec<String>,
     /// How to handle crates that have a security vulnerability
     #[serde(default = "crate::lint_deny")]
     pub vulnerability: LintLevel,
@@ -44,7 +50,9 @@ impl Default for Config {
     fn default() -> Self {
         Self {
             db_path: None,
+            db_paths: Vec::new(),
             db_url: None,
+            db_urls: Vec::new(),
             ignore: Vec::new(),
             vulnerability: LintLevel::Deny,
             unmaintained: LintLevel::Warn,
@@ -62,10 +70,22 @@ impl crate::cfg::UnvalidatedConfig for Config {
         let mut ignored: Vec<_> = self.ignore.into_iter().map(AdvisoryId::from).collect();
         ignored.sort();
 
+        let mut db_urls = self.db_urls;
+        if let Some(db_url) = self.db_url {
+            log::warn!("the 'db_url' option is deprecated, use 'db_urls' instead");
+            db_urls.push(db_url);
+        }
+
+        let mut db_paths = self.db_paths;
+        if let Some(db_path) = self.db_path {
+            log::warn!("the 'db_path' option is deprecated, use 'db_paths' instead");
+            db_paths.push(db_path);
+        }
+
         Ok(ValidConfig {
             file_id: cfg_file,
-            db_path: self.db_path,
-            db_url: self.db_url,
+            db_paths,
+            db_urls,
             ignore: ignored,
             vulnerability: self.vulnerability,
             unmaintained: self.unmaintained,
@@ -80,8 +100,8 @@ pub(crate) type AdvisoryId = Spanned<advisory::Id>;
 
 pub struct ValidConfig {
     pub file_id: FileId,
-    pub db_path: Option<PathBuf>,
-    pub db_url: Option<String>,
+    pub db_paths: Vec<PathBuf>,
+    pub db_urls: Vec<String>,
     pub(crate) ignore: Vec<AdvisoryId>,
     pub vulnerability: LintLevel,
     pub unmaintained: LintLevel,
@@ -94,6 +114,7 @@ pub struct ValidConfig {
 mod test {
     use super::*;
     use crate::cfg::{test::*, UnvalidatedConfig};
+    use std::borrow::Cow;
 
     #[test]
     fn works() {
@@ -107,14 +128,15 @@ mod test {
         let validated = cd.config.advisories.validate(cd.id).unwrap();
 
         assert_eq!(validated.file_id, cd.id);
-        assert_eq!(
-            validated.db_path.as_ref().map(|dp| dp.to_string_lossy()),
-            Some(std::borrow::Cow::Borrowed("~/.cargo/advisory-db"))
-        );
-        assert_eq!(
-            validated.db_url.as_deref(),
-            Some("https://github.com/RustSec/advisory-db")
-        );
+        assert!(validated
+            .db_paths
+            .iter()
+            .map(|dp| dp.to_string_lossy())
+            .eq(vec![Cow::Borrowed("~/.cargo/advisory-db")]));
+        assert!(validated
+            .db_urls
+            .iter()
+            .eq(vec!["https://github.com/RustSec/advisory-db"]));
         assert_eq!(validated.vulnerability, LintLevel::Deny);
         assert_eq!(validated.unmaintained, LintLevel::Warn);
         assert_eq!(validated.yanked, LintLevel::Warn);

src/advisories/mod.rs

@@ -7,7 +7,10 @@ use crate::{
 use anyhow::{Context, Error};
 use log::debug;
 pub use rustsec::{advisory::Id, lockfile::Lockfile, Database};
-use rustsec::{repository as repo, Repository};
+use rustsec::{
+    database::{scope::Package, Query},
+    repository as repo, Advisory, Repository, Vulnerability,
+};
 use std::path::{Path, PathBuf};
 
 /// Whether the database will be fetched or not
@@ -17,30 +20,109 @@ pub enum Fetch {
     Disallow,
 }
 
-pub fn load_db(
-    db_url: Option<&str>,
-    db_path: Option<PathBuf>,
+/// A collection of [`Database`]s that is used to query advisories
+/// in many different databases.
+///
+/// It mirrors the API of a single [`Database`], but will query all databases.
+///
+/// [`Database`]: https://docs.rs/rustsec/0.21.0/rustsec/database/struct.Database.html
+pub struct DatabaseCollection {
+    dbs: Vec<Database>,
+}
+
+impl DatabaseCollection {
+    pub fn new(dbs: Vec<Database>) -> Self {
+        Self { dbs }
+    }
+
+    pub fn get(&self, id: &Id) -> Option<&Advisory> {
+        self.dbs.iter().flat_map(|db| db.get(id)).next()
+    }
+
+    pub fn query(&self, query: &Query) -> Vec<&Advisory> {
+        let mut results = self
+            .dbs
+            .iter()
+            .flat_map(|db| db.query(query))
+            .collect::<Vec<_>>();
+        results.dedup();
+        results
+    }
+
+    pub fn query_vulnerabilities(
+        &self,
+        lockfile: &Lockfile,
+        query: &Query,
+        package_scope: impl Into<Package>,
+    ) -> Vec<Vulnerability> {
+        let package_scope = package_scope.into();
+        let mut results = self
+            .dbs
+            .iter()
+            .flat_map(|db| db.query_vulnerabilities(lockfile, query, package_scope.clone()))
+            .collect::<Vec<_>>();
+        results.dedup();
+        results
+    }
+
+    pub fn vulnerabilities(&self, lockfile: &Lockfile) -> Vec<Vulnerability> {
+        let mut results = self
+            .dbs
+            .iter()
+            .flat_map(|db| db.vulnerabilities(lockfile))
+            .collect::<Vec<_>>();
+        results.dedup();
+        results
+    }
+
+    pub fn iter(&self) -> std::slice::Iter<'_, Database> {
+        self.dbs.iter()
+    }
+}
+
+pub fn load_dbs(
+    mut db_urls: Vec<&str>,
+    mut db_paths: Vec<PathBuf>,
+    fetch: Fetch,
+) -> Result<DatabaseCollection, Error> {
+    db_urls.dedup();
+    if db_urls.is_empty() {
+        db_urls.push(repo::DEFAULT_URL);
+    }
+
+    db_paths.dedup();
+    if db_paths.is_empty() {
+        db_paths.push(Repository::default_path());
+    }
+
+    let dbs = db_urls
+        .into_iter()
+        .zip(db_paths)
+        .map(|(url, path)| load_db(&url, path, fetch))
+        .collect::<Result<Vec<_>, Error>>()?;
+
+    Ok(DatabaseCollection::new(dbs))
+}
+
+fn load_db(
+    advisory_db_url: &str,
+    advisory_db_path: PathBuf,
     fetch: Fetch,
 ) -> Result<Database, Error> {
-    let advisory_db_url = db_url.unwrap_or(repo::DEFAULT_URL);
-    let advisory_db_path = db_path
-        .and_then(|path| {
-            if path.starts_with("~") {
-                match home::home_dir() {
-                    Some(home) => Some(home.join(path.strip_prefix("~").unwrap())),
-                    None => {
-                        log::warn!(
-                            "unable to resolve path '{}', falling back to the default advisory path",
-                            path.display()
-                        );
-                        None
-                    }
-                }
-            } else {
-                Some(path)
+    let advisory_db_path = if advisory_db_path.starts_with("~") {
+        match home::home_dir() {
+            Some(home) => home.join(advisory_db_path.strip_prefix("~").unwrap()),
+            None => {
+                log::warn!(
+                    "unable to resolve path '{}', falling back to the default advisory path",
+                    advisory_db_path.display()
+                );
+                Repository::default_path()
             }
-        })
-        .unwrap_or_else(Repository::default_path);
+        }
+    } else {
+        advisory_db_path
+    };
 
     let advisory_db_repo = match fetch {
         Fetch::Allow => {
@@ -116,7 +198,7 @@ fn krate_for_pkg<'a>(
 /// unmaintained crates
 pub fn check(
     ctx: crate::CheckCtx<'_, cfg::ValidConfig>,
-    advisory_db: &Database,
+    advisory_dbs: &DatabaseCollection,
     mut lockfile: rustsec::lockfile::Lockfile,
     sender: crossbeam::channel::Sender<diag::Pack>,
 ) {
@@ -148,8 +230,13 @@ pub fn check(
         .packages
         .retain(|pkg| krate_for_pkg(&ctx.krates, pkg).is_some());
 
-    let (report, yanked) = rayon::join(
-        || rustsec::Report::generate(&advisory_db, &lockfile, &settings),
+    let (reports, yanked) = rayon::join(
+        || {
+            advisory_dbs
+                .iter()
+                .map(|db| rustsec::Report::generate(db, &lockfile, &settings))
+                .collect::<Vec<_>>()
+        },
         || {
             let index = rustsec::registry::Index::open()?;
             let mut yanked = Vec::new();
@@ -278,18 +365,21 @@ pub fn check(
     };
 
     // Check if any vulnerabilities were found
-    if report.vulnerabilities.found {
-        for vuln in &report.vulnerabilities.list {
+    if reports.iter().any(|report| report.vulnerabilities.found) {
+        for vuln in reports
+            .iter()
+            .flat_map(|report| &report.vulnerabilities.list)
+        {
             sender
                 .send(make_diag(&vuln.package, &vuln.advisory))
                 .unwrap();
         }
     }
 
     // Check for informational advisories for crates, including unmaintained
-    for (_kind, warnings) in report.warnings {
+    for (_kind, warnings) in reports.iter().flat_map(|report| &report.warnings) {
         for warning in warnings {
-            if let Some(advisory) = warning.advisory {
+            if let Some(advisory) = &warning.advisory {
                 let diag = make_diag(&warning.package, &advisory);
                 sender.send(diag).unwrap();
             }

src/cargo-deny/check.rs

@@ -231,9 +231,13 @@ pub(crate) fn cmd(
 
         if check_advisories {
             s.spawn(|_| {
-                advisory_db = Some(advisories::load_db(
-                    cfg.advisories.db_url.as_ref().map(AsRef::as_ref),
-                    cfg.advisories.db_path.as_ref().cloned(),
+                advisory_db = Some(advisories::load_dbs(
+                    cfg.advisories.db_urls.iter().map(AsRef::as_ref).collect(),
+                    cfg.advisories
+                        .db_paths
+                        .iter()
+                        .map(|p| p.to_path_buf())
+                        .collect(),
                     if args.disable_fetch {
                         advisories::Fetch::Disallow
                     } else {

src/cargo-deny/fetch.rs

@@ -142,9 +142,9 @@ pub fn cmd(
         if fetch_db {
             s.spawn(|_| {
                 // This function already logs internally
-                db = Some(advisories::load_db(
-                    cfg.advisories.db_url.as_ref().map(AsRef::as_ref),
-                    cfg.advisories.db_path.as_ref().cloned(),
+                db = Some(advisories::load_dbs(
+                    cfg.advisories.db_urls.iter().map(AsRef::as_ref).collect(),
+                    cfg.advisories.db_paths,
                     advisories::Fetch::Allow,
                 ))
             });