Additional config options for the HTTP client. (#111)

This PR adds four new configuration options for the HTTP client used by the
json unit.

These are http-root-certs for additional TLS root certificates,
http-user-agent for setting a custom user agent, http-client-addr to specify
a local address to bind to, and http-proxies to add HTTP proxies. The last
option is only available if the socks feature is enabled which it is by
default.

Cargo.toml

@@ -36,6 +36,11 @@ toml            = "0.8.2"
 url             = { version = "2.2", features = ["serde"] }
 webpki-roots    = "0.25.2"
 
+[features]
+default = [ "socks" ]
+socks = [ "reqwest/socks" ]
+
+
 [dev-dependencies]
 stderrlog       = "0.6"
 rand_pcg        = "0.3"

doc/manual/source/configuration.rst

@@ -44,6 +44,21 @@ Note that details are provided for each unit and each target.
     # Where should the HTTP server listen on?
     http-listen = ["127.0.0.1:8080"]
 
+    # The proxy servers to use for outgoing HTTP requests.
+    #
+    # Note: This option is only used if RTRTR is built with the socks feature
+    # enabled. This is true by default.
+    http-proxies = [ "socks5://192.168.1.3:9000" ]
+
+    # Additional root certificates for outgoing HTTP requests
+    http-root-certs = [ "/var/lib/rtrtr/root-cert/some.crt" ]
+
+    # The user agent string to use for outgoing HTTP requests.
+    http-user-agent = "My RPKI proxy"
+
+    # Local address to bind to for outgoing HTTP requests.
+    http-client-addr = "198.168.1.2"
+
 Units
 -----
 

src/config.rs

@@ -15,7 +15,7 @@ use daemonbase::error::Failed;
 use serde::Deserialize;
 use toml::Spanned;
 use crate::http;
-use crate::manager::{Manager, TargetSet, UnitSet};
+use crate::manager::{HttpClientConfig, Manager, TargetSet, UnitSet};
 
 
 //------------ Config --------------------------------------------------------
@@ -45,6 +45,10 @@ pub struct Config {
     /// The HTTP server configuration.
     #[serde(flatten)]
     pub http: http::Server,
+
+    /// The HTTP client configuration.
+    #[serde(flatten)]
+    pub http_client: HttpClientConfig,
 }
 
 impl Config {
@@ -75,12 +79,12 @@ impl Config {
     /// to panic.
     ///
     /// The current path needs to be provided to be able to deal with relative
-    /// paths. The manager is necessary to resolve links given in the
-    /// configuration.
+    /// paths.
+    ///
+    /// Returns both a manager and the config.
     pub fn from_arg_matches(
         matches: &clap::ArgMatches,
-        manager: &mut Manager,
-    ) -> Result<Self, Failed> {
+    ) -> Result<(Manager, Self), Failed> {
         let args = Args::from_arg_matches(
             matches
         ).expect("bug in command line arguments parser");
@@ -95,9 +99,9 @@ impl Config {
                 return Err(Failed)
             }
         };
-        let mut res = manager.load(conf)?;
-        res.log.apply_args(&args.log);
-        Ok(res)
+        let (manager, mut config) = Manager::load(conf)?;
+        config.log.apply_args(&args.log);
+        Ok((manager, config))
     }
 }
 

src/main.rs

@@ -5,7 +5,6 @@ use daemonbase::logging::Logger;
 use futures::future::pending;
 use tokio::runtime;
 use rtrtr::config::Config;
-use rtrtr::manager::Manager;
 
 
 fn _main() -> Result<(), ExitError> {
@@ -16,10 +15,7 @@ fn _main() -> Result<(), ExitError> {
         .author(crate_authors!())
         .about("collecting, processing and distributing route filtering data")
     ).get_matches();
-    let mut manager = Manager::new();
-    let mut config = Config::from_arg_matches(
-        &matches, &mut manager
-    )?;
+    let (mut manager, mut config) = Config::from_arg_matches(&matches)?;
     Logger::from_config(&config.log)?.switch_logging(false)?;
     let runtime = runtime::Builder::new_multi_thread()
         .enable_all()

src/manager.rs

@@ -1,12 +1,16 @@
 //! Controlling the entire operation.
 
+use std::{fs, io};
 use std::cell::RefCell;
 use std::collections::HashMap;
+use std::net::IpAddr;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
+use clap::crate_version;
 use daemonbase::error::Failed;
 use log::error;
 use serde::Deserialize;
-use reqwest::Client as HttpClient;
+use reqwest::{Certificate, Client as HttpClient};
 use tokio::runtime;
 use crate::{http, metrics};
 use crate::comms::{Gate, GateAgent, Link};
@@ -15,6 +19,29 @@ use crate::targets::Target;
 use crate::units::Unit;
 
 
+//------------ HttpClientConfig ----------------------------------------------
+
+#[derive(Clone, Debug, Default, Deserialize)]
+pub struct HttpClientConfig {
+    /// The proxy servers to use for outgoing HTTP requests.
+    #[cfg(feature = "socks")]
+    #[serde(default, rename = "http-proxies")]
+    proxies: Vec<String>,
+
+    /// Additional root certificates for outgoing HTTP requests.
+    #[serde(default, rename = "http-root-certs")]
+    root_certs: Vec<PathBuf>,
+
+    /// The user agent string to use for outgoing HTTP requests.
+    #[serde(rename = "http-user-agent")]
+    user_agent: Option<String>,
+
+    /// Local address to bind to for outgoing HTTP requests.
+    #[serde(rename = "http-client-addr")]
+    local_addr: Option<IpAddr>,
+}
+
+
 //------------ Component -----------------------------------------------------
 
 /// Facilities available to all components.
@@ -97,8 +124,81 @@ pub struct Manager {
 
 impl Manager {
     /// Creates a new manager.
-    pub fn new() -> Self {
-        Default::default()
+    pub fn new(http_config: &HttpClientConfig) -> Result<Self, Failed> {
+        let mut builder = HttpClient::builder();
+
+        #[cfg(feature = "socks")]
+        for proxy in &http_config.proxies {
+            let proxy = match reqwest::Proxy::all(proxy) {
+                Ok(proxy) => proxy,
+                Err(err) => {
+                    error!(
+                        "Invalid rrdp-proxy '{}': {}", proxy, err
+                    );
+                    return Err(Failed)
+                }
+            };
+            builder = builder.proxy(proxy);
+        }
+
+        for path in &http_config.root_certs {
+            builder = builder.add_root_certificate(
+                Self::load_cert(path)?
+            );
+        }
+
+        builder = builder.user_agent(
+            match http_config.user_agent.as_ref() {
+                Some(agent) => agent.as_str(),
+                None => concat!("RTRTR ", crate_version!()),
+            }
+        );
+
+        if let Some(addr) = http_config.local_addr {
+            builder = builder.local_address(addr)
+        }
+
+        let client = match builder.build() {
+            Ok(client) => client,
+            Err(err) => {
+                error!("Failed to initialize HTTP client: {}.", err);
+                return Err(Failed)
+            }
+        };
+
+        Ok(Manager {
+            http_client: client,
+            .. Default::default()
+        })
+    }
+
+    /// Loads a WebPKI trusted certificate.
+    fn load_cert(path: &Path) -> Result<Certificate, Failed> {
+        let mut file = match fs::File::open(path) {
+            Ok(file) => file,
+            Err(err) => {
+                error!(
+                    "Cannot open rrdp-root-cert file '{}': {}'",
+                    path.display(), err
+                );
+                return Err(Failed);
+            }
+        };
+        let mut data = Vec::new();
+        if let Err(err) = io::Read::read_to_end(&mut file, &mut data) {
+            error!(
+                "Cannot read rrdp-root-cert file '{}': {}'",
+                path.display(), err
+            );
+            return Err(Failed);
+        }
+        Certificate::from_pem(&data).map_err(|err| {
+            error!(
+                "Cannot decode rrdp-root-cert file '{}': {}'",
+                path.display(), err
+            );
+            Failed
+        })
     }
 
     /// Loads the given config file.
@@ -111,17 +211,15 @@ impl Manager {
     ///
     /// If the method succeeds, you need to spawn all units and targets via
     /// the [`spawn`](Self::spawn) method.
+    ///
+    /// Returns both a new manager and the parsed config.
     pub fn load(
-        &mut self, file: ConfigFile
-    ) -> Result<Config, Failed> {
+        file: ConfigFile
+    ) -> Result<(Self, Config), Failed> {
         // Prepare the thread-local used to allow serde load the links in the
         // units and targets.
         GATES.with(|gates| {
-            gates.replace(
-                Some(self.units.iter().map(|(key, value)| {
-                    (key.clone(), value.clone().into())
-                }).collect())
-            )
+            gates.replace(Some(Default::default()))
         });
 
         // Now load the config file.
@@ -136,6 +234,8 @@ impl Manager {
             }
         };
 
+        let mut manager = Self::new(&config.http_client)?;
+
         // All entries in the thread-local that have a gate are new. They must
         // appear in config’s units or we have unresolved links.
         let gates = GATES.with(|gates| gates.replace(None) ).unwrap();
@@ -151,8 +251,8 @@ impl Manager {
                     }
                 }
                 else {
-                    self.units.insert(name.clone(), load.agent);
-                    self.pending.insert(name, gate);
+                    manager.units.insert(name.clone(), load.agent);
+                    manager.pending.insert(name, gate);
                 }
             }
         }
@@ -163,7 +263,7 @@ impl Manager {
             return Err(Failed)
         }
 
-        Ok(config)
+        Ok((manager, config))
     }
 
     /// Allows creating components and adding them to the manager.

src/test.rs

@@ -154,7 +154,7 @@ async fn simple_comms() {
     use crate::manager::Manager;
     use crate::payload::testrig;
 
-    let mut manager = Manager::new();
+    let mut manager = Manager::default();
 
     let (u, mut t) = manager.add_components(
         &runtime::Handle::current(),

src/units/combine.rs

@@ -208,7 +208,7 @@ mod test {
     #[tokio::test]
     async fn wake_up_again() {
         test::init_log();
-        let mut manager = Manager::new();
+        let mut manager = Manager::default();
 
         let (u1, u2, u3, mut t) = manager.add_components(
             &runtime::Handle::current(),