refactor: pull up the new_source function to allow re-using them

common/src/cli/client.rs

@@ -12,13 +12,18 @@ pub struct ClientArguments {
     pub retries: usize,
 }
 
+impl From<ClientArguments> for FetcherOptions {
+    fn from(value: ClientArguments) -> Self {
+        FetcherOptions {
+            timeout: value.timeout.into(),
+            retries: value.retries,
+        }
+    }
+}
+
 impl ClientArguments {
     /// Create a new [`Fetcher`] from arguments.
     pub async fn new_fetcher(self) -> Result<Fetcher, anyhow::Error> {
-        Fetcher::new(FetcherOptions {
-            timeout: self.timeout.into(),
-            retries: self.retries,
-        })
-        .await
+        Fetcher::new(self.into()).await
     }
 }

common/src/sender/provider/openid.rs

@@ -1,7 +1,6 @@
 use super::{Credentials, Expires, TokenProvider};
-use crate::{sender::Error, utils::pem::add_cert};
+use crate::sender::Error;
 use core::fmt::{self, Debug, Formatter};
-use std::path::PathBuf;
 use std::{ops::Deref, sync::Arc};
 use tokio::sync::RwLock;
 
@@ -52,7 +51,7 @@ pub struct OpenIdTokenProviderConfigArguments {
         long = "oidc-tls-ca-certificate",
         action = clap::ArgAction::Append,
     )]
-    pub tls_ca_certificates: Vec<PathBuf>,
+    pub tls_ca_certificates: Vec<std::path::PathBuf>,
 }
 
 #[cfg(feature = "clap")]
@@ -70,7 +69,7 @@ pub struct OpenIdTokenProviderConfig {
     pub issuer_url: String,
     pub refresh_before: humantime::Duration,
     pub tls_insecure: bool,
-    pub tls_ca_certificates: Vec<PathBuf>,
+    pub tls_ca_certificates: Vec<std::path::PathBuf>,
 }
 
 #[cfg(feature = "clap")]
@@ -154,7 +153,7 @@ impl OpenIdTokenProvider {
         }
 
         for cert in config.tls_ca_certificates {
-            client = add_cert(client, &cert)
+            client = crate::utils::pem::add_cert(client, &cert)
                 .with_context(|| format!("adding trust anchor: {}", cert.display()))?;
         }
 

common/src/utils/pem.rs

@@ -1,6 +1,7 @@
 use reqwest::Certificate;
 use std::{fs::File, io::Read, path::Path};
 
+#[allow(unused)]
 pub fn add_cert<P: AsRef<Path>>(
     mut client: reqwest::ClientBuilder,
     cert: P,

csaf/Cargo.toml

@@ -21,6 +21,7 @@ cpe = "0.1.3"
 csv = "1"
 digest = "0.10.6"
 filetime = "0.2"
+fluent-uri = "0.1.4"
 futures = "0.3"
 hickory-resolver = { version = "0.24.0", features = ["tokio-runtime"] }
 humantime = "2"

csaf/csaf-cli/Cargo.toml

@@ -20,7 +20,6 @@ colored_json = "4"
 csaf = { version = "0.5.0", default-features = false }
 env_logger = "0.11.2"
 flexible-time = "0.1.1"
-fluent-uri = "0.1.4"
 humantime = "2"
 indicatif = { version = "0.17.3", features = [] }
 indicatif-log-bridge = "0.2.1"

csaf/csaf-cli/src/cmd/discover.rs

@@ -1,7 +1,8 @@
 use crate::{
     cmd::{DiscoverArguments, FilterArguments},
-    common::{filter, new_source},
+    common::filter,
 };
+use csaf_walker::source::new_source;
 use csaf_walker::{discover::DiscoveredAdvisory, walker::Walker};
 use std::convert::Infallible;
 use walker_common::{cli::client::ClientArguments, progress::Progress};

csaf/csaf-cli/src/cmd/download.rs

@@ -1,8 +1,9 @@
 use crate::{
     cmd::{DiscoverArguments, FilterArguments, SkipArguments, StoreArguments},
-    common::{walk_visitor, DiscoverConfig},
+    common::walk_visitor,
 };
 use csaf_walker::{
+    discover::DiscoverConfig,
     retrieve::RetrievingVisitor,
     visitors::{skip::SkipExistingVisitor, store::StoreVisitor},
 };

csaf/csaf-cli/src/cmd/metadata.rs

@@ -1,9 +1,9 @@
-use crate::common::{new_source, DiscoverConfig};
 use colored_json::write_colored_json;
 use csaf_walker::{
+    discover::DiscoverConfig,
     metadata::{self, MetadataRetriever},
     model::metadata::ProviderMetadata,
-    source::Source,
+    source::{new_source, Source},
 };
 use std::fmt::Display;
 use std::io::stdout;

csaf/csaf-cli/src/cmd/mod.rs

@@ -17,10 +17,6 @@ pub mod sync;
 pub struct DiscoverArguments {
     /// Source to scan from, will be suffixed with "/.well-known/csaf/provider-metadata.json" unless "--full" is used.
     pub source: String,
-
-    #[arg(long)]
-    /// Treat the "source" as a full URL to the metadata.
-    pub full: bool,
 }
 
 #[derive(Debug, clap::Parser)]

csaf/csaf-cli/src/cmd/send.rs

@@ -1,9 +1,10 @@
 use crate::{
     cmd::{DiscoverArguments, FilterArguments, SkipArguments},
-    common::{walk_visitor, DiscoverConfig},
+    common::walk_visitor,
 };
 use csaf_walker::{
-    retrieve::RetrievingVisitor, validation::ValidationVisitor, visitors::skip::SkipFailedVisitor,
+    discover::DiscoverConfig, retrieve::RetrievingVisitor, validation::ValidationVisitor,
+    visitors::skip::SkipFailedVisitor,
 };
 use walker_common::{
     cli::{client::ClientArguments, runner::RunnerArguments, validation::ValidationArguments},

csaf/csaf-cli/src/cmd/sync.rs

@@ -1,7 +1,8 @@
 use crate::{
     cmd::{DiscoverArguments, FilterArguments, SkipArguments, StoreArguments},
-    common::{walk_visitor, DiscoverConfig},
+    common::walk_visitor,
 };
+use csaf_walker::discover::DiscoverConfig;
 use csaf_walker::{
     retrieve::RetrievingVisitor,
     validation::ValidationVisitor,

csaf/csaf-cli/src/common.rs

@@ -1,19 +1,13 @@
 use crate::cmd::DiscoverArguments;
-use anyhow::bail;
 use csaf_walker::{
-    discover::DiscoveredVisitor,
-    metadata::MetadataRetriever,
+    discover::{DiscoverConfig, DiscoveredVisitor},
     retrieve::RetrievingVisitor,
-    source::{DispatchSource, FileOptions, FileSource, HttpOptions, HttpSource},
+    source::{new_source, DispatchSource},
     validation::{ValidatedVisitor, ValidationVisitor},
     visitors::filter::{FilterConfig, FilteringVisitor},
     walker::Walker,
 };
-use fluent_uri::Uri;
-use reqwest::Url;
 use std::future::Future;
-use std::path::PathBuf;
-use std::time::SystemTime;
 use walker_common::{
     cli::{client::ClientArguments, runner::RunnerArguments, validation::ValidationArguments},
     progress::Progress,
@@ -51,25 +45,6 @@ where
     .await
 }
 
-pub struct DiscoverConfig {
-    /// The source to locate the provider metadata.
-    ///
-    /// This can be either a full path to a provider-metadata.json, or a base domain used by the
-    /// CSAF metadata discovery process.
-    pub source: String,
-
-    /// Only report documents which have changed since the provided date. If a document has no
-    /// change information, or this field is [`None`], it will always be reported.
-    pub since: Option<SystemTime>,
-}
-
-impl DiscoverConfig {
-    pub fn with_since(mut self, since: impl Into<Option<SystemTime>>) -> Self {
-        self.since = since.into();
-        self
-    }
-}
-
 impl From<DiscoverArguments> for DiscoverConfig {
     fn from(value: DiscoverArguments) -> Self {
         Self {
@@ -79,76 +54,6 @@ impl From<DiscoverArguments> for DiscoverConfig {
     }
 }
 
-impl From<&str> for DiscoverConfig {
-    fn from(value: &str) -> Self {
-        Self {
-            since: None,
-            source: value.to_string(),
-        }
-    }
-}
-
-#[derive(Clone, Debug)]
-enum SourceDescriptor {
-    File(PathBuf),
-    Url(Url),
-    Lookup(String),
-}
-
-impl SourceDescriptor {
-    pub fn from(source: &str) -> anyhow::Result<Self> {
-        match Uri::parse(source) {
-            Ok(uri) => match uri.scheme().map(|s| s.as_str()) {
-                Some("https") => Ok(SourceDescriptor::Url(Url::parse(source)?)),
-                Some("file") => Ok(SourceDescriptor::File(PathBuf::from(uri.path().as_str()))),
-                Some(other) => bail!("URLs with scheme '{other}' are not supported"),
-                None => Ok(SourceDescriptor::Lookup(source.to_string())),
-            },
-            Err(err) => {
-                log::debug!("Failed to handle source as URL: {err}");
-                Ok(SourceDescriptor::Lookup(source.to_string()))
-            }
-        }
-    }
-
-    pub async fn into_source(
-        self,
-        discover: DiscoverConfig,
-        client: ClientArguments,
-    ) -> anyhow::Result<DispatchSource> {
-        match self {
-            Self::File(path) => {
-                Ok(FileSource::new(path, FileOptions::new().since(discover.since))?.into())
-            }
-            Self::Url(url) => Ok(HttpSource::new(
-                url,
-                client.new_fetcher().await?,
-                HttpOptions::new().since(discover.since),
-            )
-            .into()),
-            Self::Lookup(source) => {
-                let fetcher = client.new_fetcher().await?;
-                Ok(HttpSource::new(
-                    MetadataRetriever::new(source),
-                    fetcher,
-                    HttpOptions::new().since(discover.since),
-                )
-                .into())
-            }
-        }
-    }
-}
-
-pub async fn new_source(
-    discover: impl Into<DiscoverConfig>,
-    client: ClientArguments,
-) -> anyhow::Result<DispatchSource> {
-    let discover = discover.into();
-
-    let descriptor = SourceDescriptor::from(&discover.source)?;
-    descriptor.into_source(discover, client).await
-}
-
 /// Create a [`FilteringVisitor`] from a [`FilterConfig`].
 pub fn filter<V>(filter: impl Into<FilterConfig>, visitor: V) -> FilteringVisitor<V>
 where
@@ -212,11 +117,12 @@ where
 #[cfg(test)]
 mod test {
 
-    use super::*;
+    use csaf_walker::source::SourceDescriptor;
+    use std::str::FromStr;
 
     #[tokio::test]
     async fn test_file_relative() {
-        let source = SourceDescriptor::from("file:../../foo/bar");
+        let source = SourceDescriptor::from_str("file:../../foo/bar");
         println!("Result: {source:?}");
         assert!(
             matches!(source, Ok(SourceDescriptor::File(path)) if path.to_string_lossy() == "../../foo/bar")
@@ -225,7 +131,7 @@ mod test {
 
     #[tokio::test]
     async fn test_file_absolute() {
-        let source = SourceDescriptor::from("file:/foo/bar");
+        let source = SourceDescriptor::from_str("file:/foo/bar");
         println!("Result: {source:?}");
         assert!(
             matches!(source, Ok(SourceDescriptor::File(path)) if path.to_string_lossy() == "/foo/bar")
@@ -234,7 +140,7 @@ mod test {
 
     #[tokio::test]
     async fn test_file_absolute_double() {
-        let source = SourceDescriptor::from("file:///foo/bar");
+        let source = SourceDescriptor::from_str("file:///foo/bar");
         println!("Result: {source:?}");
         assert!(
             matches!(source, Ok(SourceDescriptor::File(path)) if path.to_string_lossy() == "/foo/bar")
@@ -243,7 +149,7 @@ mod test {
 
     #[tokio::test]
     async fn test_file_absolute_windows() {
-        let source = SourceDescriptor::from("file:///c:/DATA/example");
+        let source = SourceDescriptor::from_str("file:///c:/DATA/example");
         println!("Result: {source:?}");
         assert!(
             matches!(source, Ok(SourceDescriptor::File(path)) if path.to_string_lossy() == "/c:/DATA/example")
@@ -252,14 +158,14 @@ mod test {
 
     #[tokio::test]
     async fn test_base() {
-        let source = SourceDescriptor::from("base.domain");
+        let source = SourceDescriptor::from_str("base.domain");
         println!("Result: {source:?}");
         assert!(matches!(source, Ok(SourceDescriptor::Lookup(base)) if base == "base.domain"));
     }
 
     #[tokio::test]
     async fn test_https() {
-        let source = SourceDescriptor::from("https://base.domain");
+        let source = SourceDescriptor::from_str("https://base.domain");
         println!("Result: {source:?}");
         assert!(
             matches!(source, Ok(SourceDescriptor::Url(url)) if url.as_str() == "https://base.domain/")
@@ -268,7 +174,7 @@ mod test {
 
     #[tokio::test]
     async fn test_gopher() {
-        let source = SourceDescriptor::from("gopher://base.domain");
+        let source = SourceDescriptor::from_str("gopher://base.domain");
         println!("Result: {source:?}");
         assert!(source.is_err());
     }

csaf/src/discover.rs

@@ -9,6 +9,35 @@ use std::time::SystemTime;
 use url::Url;
 use walker_common::utils::url::Urlify;
 
+/// Discovery configuration
+pub struct DiscoverConfig {
+    /// The source to locate the provider metadata.
+    ///
+    /// This can be either a full path to a provider-metadata.json, or a base domain used by the
+    /// CSAF metadata discovery process.
+    pub source: String,
+
+    /// Only report documents which have changed since the provided date. If a document has no
+    /// change information, or this field is [`None`], it will always be reported.
+    pub since: Option<SystemTime>,
+}
+
+impl DiscoverConfig {
+    pub fn with_since(mut self, since: impl Into<Option<SystemTime>>) -> Self {
+        self.since = since.into();
+        self
+    }
+}
+
+impl From<&str> for DiscoverConfig {
+    fn from(value: &str) -> Self {
+        Self {
+            since: None,
+            source: value.to_string(),
+        }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum DistributionContext {
     Directory(Url),