Build scripts and shenanigans (#549)

- Fix committer name
- Update krates/tame-index/gix
- Checkpoint
- Update dependencies

Resolves: #43 
Resolves: #548 
Resolves: #552
Resolves: #553 (I guess? It adds a feature toggle for using the OS
certificate store)

Cargo.toml

@@ -25,6 +25,10 @@ name = "cargo-deny"
 path = "src/cargo-deny/main.rs"
 
 [features]
+default = ["reqwest/rustls-tls-webpki-roots", "tame-index/default"]
+# Enables the use of OS native certificate store.
+native-certs = ["reqwest/rustls-tls-native-roots", "tame-index/native-certs"]
+
 #default = ["vendored-openssl", "vendored-libgit2"]
 # Allows the use of a vendored version openssl when compiling libgit, which allows
 # us to compile static executables (eg musl) and avoid system dependencies
@@ -49,26 +53,42 @@ camino = "1.1"
 # Allows us to do eg cargo metadata operations without relying on an external cargo
 #cargo = { version = "0.71", optional = true }
 # Argument parsing, kept aligned with cargo
-clap = { version = "4.0", features = ["derive", "env"] }
+clap = { version = "4.3", features = ["derive", "env"] }
 # Used for diagnostic reporting
 codespan = "0.11"
 codespan-reporting = "0.11"
 # Brrrrr
 crossbeam = "0.8"
 # Logging utilities
 fern = "0.6"
+# Glob matching
+globset = "0.4"
+# Native executable detection
+goblin = { version = "0.7", default-features = false, features = [
+  "elf32",
+  "elf64",
+  "mach32",
+  "mach64",
+  "pe32",
+  "pe64",
+] }
 # We need to figure out HOME/CARGO_HOME in some cases
 home = "0.5"
 # Provides graphs on top of cargo_metadata
-krates = { version = "0.14", features = ["prefer-index", "targets"] }
+krates = { version = "0.15", features = ["targets"] }
 # Log macros
 log = "0.4"
 # Nicer sync primitives
 parking_lot = "0.12"
 # Moar brrrr
 rayon = "1.4"
+# HTTP client backing gix, we don't use it directly but need it here for configuration
+# due to ...reasons
+reqwest = { version = "0.11", default-features = false }
+# sha-256 hash calculation, already a dependency via rustls/etc
+ring = "0.16"
 # Used for interacting with advisory databases
-rustsec = { version = "0.27", default-features = false }
+rustsec = { version = "0.28", default-features = false }
 # Parsing and checking of versions/version requirements
 semver = "1.0"
 # Gee what could it be
@@ -81,7 +101,10 @@ spdx = "0.10"
 # Lazy
 strum = { version = "0.25", features = ["derive"] }
 # Index retrieval and querying
-tame-index = { version = "0.2", features = ["git", "sparse"] }
+tame-index = { version = "0.5", default-features = false, features = [
+  "git",
+  "sparse",
+] }
 # Timestamp emission
 time = { version = "0.3", default-features = false, features = [
   "formatting",
@@ -93,23 +116,26 @@ toml = "0.7"
 twox-hash = { version = "1.5", default-features = false }
 # Url parsing/manipulation
 url = "2.1"
+# Directory traversal
+walkdir = "2.3"
 
 # We clone/fetch advisory databases
 [dependencies.gix]
-version = "0.50"
+version = "0.52"
 default-features = false
 features = [
-  "blocking-http-transport-reqwest-rust-tls",
+  "blocking-http-transport-reqwest",
   "blocking-network-client",
   "max-performance-safe",
+  "reqwest-for-configuration-only",
 ]
 
 [dev-dependencies]
 # Folder copying
 fs_extra = "1.3"
 # Snapshot testing
 insta = { version = "1.21", features = ["json"] }
-tame-index = { version = "0.2", features = ["local-builder"] }
+tame-index = { version = "0.5", features = ["local-builder"] }
 # We use this for creating fake crate directories for crawling license files on disk
 tempfile = "3.1.0"
 

deny.toml

@@ -32,16 +32,14 @@ deny = [
 skip = [
     # indexmap + imara-diff
     { name = "hashbrown", version = "=0.12.3" },
-    # trust-dns-proto uses an old version
-    { name = "idna", version = "=0.2.3" },
     # petgraph + h2 use an old version
     { name = "indexmap", version = "=1.9.3" },
     # hyper/tokio uses an old version
     { name = "socket2", version = "=0.4.9" },
     # strum_macros + maybe-async
     { name = "syn", version = "=1.0.109" },
-    # reqwest uses an old version
-    { name = "winreg", version = "=0.10.1" },
+    # security-framework uses an ooooold version
+    { name = "bitflags", version = "=1.3.2" },
 ]
 skip-tree = [
     # Sigh
@@ -70,9 +68,15 @@ allow = [
     "ISC",
 ]
 exceptions = [
-    { allow = ["Zlib"], name = "tinyvec" },
-    { allow = ["Unicode-DFS-2016"], name = "unicode-ident" },
-    { allow = ["OpenSSL"], name = "ring" },
+    { allow = [
+        "Zlib",
+    ], name = "tinyvec" },
+    { allow = [
+        "Unicode-DFS-2016",
+    ], name = "unicode-ident" },
+    { allow = [
+        "OpenSSL",
+    ], name = "ring" },
 ]
 
 # Sigh

src/advisories.rs

@@ -88,7 +88,10 @@ pub fn check<R, S>(
     }
 
     for (krate, status) in yanked {
-        let Some(ind) = ctx.krates.nid_for_kid(&krate.id) else { log::warn!("failed to locate node id for '{krate}'"); continue };
+        let Some(ind) = ctx.krates.nid_for_kid(&krate.id) else {
+            log::warn!("failed to locate node id for '{krate}'");
+            continue;
+        };
 
         if let Some(e) = status {
             if ctx.cfg.yanked.value != LintLevel::Allow {

src/advisories/cfg.rs

@@ -85,7 +85,12 @@ impl Default for Config {
 impl crate::cfg::UnvalidatedConfig for Config {
     type ValidCfg = ValidConfig;
 
-    fn validate(self, cfg_file: FileId, diags: &mut Vec<Diagnostic>) -> Self::ValidCfg {
+    fn validate(
+        self,
+        cfg_file: FileId,
+        _files: &mut crate::diag::Files,
+        diags: &mut Vec<Diagnostic>,
+    ) -> Self::ValidCfg {
         let mut ignored: Vec<_> = self.ignore.into_iter().map(AdvisoryId::from).collect();
         ignored.sort();
 
@@ -352,9 +357,12 @@ mod test {
             advisories: Config,
         }
 
-        let cd: ConfigData<Advisories> = load("tests/cfg/advisories.toml");
+        let mut cd: ConfigData<Advisories> = load("tests/cfg/advisories.toml");
         let mut diags = Vec::new();
-        let validated = cd.config.advisories.validate(cd.id, &mut diags);
+        let validated = cd
+            .config
+            .advisories
+            .validate(cd.id, &mut cd.files, &mut diags);
         assert!(
             !diags
                 .iter()

src/advisories/helpers/db.rs

@@ -243,7 +243,7 @@ fn fetch_and_checkout(repo: &mut gix::Repository) -> anyhow::Result<()> {
     {
         let mut config = repo.config_snapshot_mut();
         config
-            .set_raw_value("committer", None, "name", "tame-index")
+            .set_raw_value("committer", None, "name", "cargo-deny")
             .context("failed to set `committer.name`")?;
         // Note we _have_ to set the email as well, but luckily gix does not actually
         // validate if it's a proper email or not :)
@@ -353,13 +353,13 @@ fn fetch_and_checkout(repo: &mut gix::Repository) -> anyhow::Result<()> {
     .with_context(|| format!("failed to create index from tree '{root_tree}'"))?;
     let mut index = gix::index::File::from_state(index, repo.index_path());
 
-    let opts = gix::worktree::checkout::Options {
+    let opts = gix::worktree::state::checkout::Options {
         destination_is_initially_empty: false,
         overwrite_existing: true,
         ..Default::default()
     };
 
-    gix::worktree::checkout(
+    gix::worktree::state::checkout(
         &mut index,
         workdir,
         {
@@ -669,6 +669,7 @@ impl<'db, 'k> Report<'db, 'k> {
                             package,
                             advisory: Some(advisory.metadata.clone()),
                             versions: Some(advisory.versions.clone()),
+                            affected: advisory.affected.clone(),
                         };
 
                         if let Some(v) = warnings.get_mut(&kind) {

src/advisories/helpers/index.rs

@@ -1,12 +1,13 @@
 use crate::{Krate, Krates, Source};
-use anyhow::Context as _;
 use rayon::prelude::{IntoParallelIterator, ParallelIterator};
-use std::{borrow::Cow, collections::BTreeMap};
+use std::collections::BTreeMap;
 use tame_index::{index::ComboIndexCache, Error, IndexLocation, IndexUrl};
 
+type YankMap = Vec<(semver::Version, bool)>;
+
 pub struct Indices<'k> {
     pub indices: Vec<(&'k Source, Result<ComboIndexCache, Error>)>,
-    pub cache: BTreeMap<(&'k str, &'k Source), tame_index::IndexKrate>,
+    pub cache: BTreeMap<(&'k str, &'k Source), YankMap>,
 }
 
 impl<'k> Indices<'k> {
@@ -57,43 +58,41 @@ impl<'k> Indices<'k> {
                     .iter()
                     .find_map(|(url, index)| index.as_ref().ok().filter(|_i| src == *url))?;
 
-                index
-                    .cached_krate(name.try_into().ok()?)
-                    .ok()?
-                    .map(|ik| ((name, src), ik))
+                index.cached_krate(name.try_into().ok()?).ok()?.map(|ik| {
+                    let yank_map = Self::load_index_krate(ik);
+                    ((name, src), yank_map)
+                })
             })
             .collect();
 
         Self { indices, cache }
     }
 
+    #[inline]
+    fn load_index_krate(ik: tame_index::IndexKrate) -> YankMap {
+        ik.versions
+            .into_iter()
+            .filter_map(|iv| Some((iv.version.parse().ok()?, iv.yanked)))
+            .collect()
+    }
+
     #[inline]
     pub fn is_yanked(&self, krate: &'k Krate) -> anyhow::Result<bool> {
+        use anyhow::Context as _;
+
         // Ignore non-registry crates when checking, as a crate sourced
         // locally or via git can have the same name as a registry package
-        let Some(src) = krate.source.as_ref().filter(|s| s.is_registry()) else { return Ok(false) };
-
-        let index_krate = if let Some(ik) = self.cache.get(&(krate.name.as_str(), src)) {
-            Cow::Borrowed(ik)
-        } else {
-            let index = self
-                .indices
-                .iter()
-                .find_map(|(url, index)| (src == *url).then_some(index.as_ref()))
-                .context("unable to find source index")?
-                .map_err(|err| anyhow::anyhow!("failed to load index: {err:#}"))?;
-
-            let ik = index
-                .cached_krate(krate.name.as_str().try_into()?)
-                .context("failed to read crate from index cache")?
-                .context("unable to find crate in cache")?;
-            Cow::Owned(ik)
+        let Some(src) = krate.source.as_ref().filter(|s| s.is_registry()) else {
+            return Ok(false);
         };
 
-        let is_yanked = index_krate
-            .versions
+        let cache_entry = self
+            .cache
+            .get(&(krate.name.as_str(), src))
+            .context("unable to locate index metadata")?;
+        let is_yanked = cache_entry
             .iter()
-            .find_map(|kv| (kv.version == krate.version).then_some(kv.yanked));
+            .find_map(|kv| (kv.0 == krate.version).then_some(kv.1));
 
         Ok(is_yanked.unwrap_or_default())
     }