feat(search): add override for validation certificates for testing pu…

…rposes
umccr · Apr 26, 2024 · a0aba09 · a0aba09
1 parent df675cd
commit a0aba09
Show file tree

Hide file tree

Showing 7 changed files with 41 additions and 124 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml
@@ -37,10 +37,11 @@ rustls-native-certs = "0.6"
 hyper-rustls = { version = "0.24", features = ["rustls-native-certs", "http2", "http1"] }
 
 hyper = { version = "0.14", features = ["http1", "http2", "client"], optional = true }
-reqwest = { version = "0.11", features = ["rustls-tls", "native-tls"], default-features = false }
+reqwest = { version = "0.11", features = ["rustls-tls"], default-features = false }
 
 async-crypt4gh = { version = "0.1.0", path = "../async-crypt4gh", default-features = false, optional = true }
 crypt4gh = { version = "0.4", git = "https://github.com/EGA-archive/crypt4gh-rust", optional = true }
+log = "0.4.21"
 
 [dev-dependencies]
 serde_json = "1.0"

diff --git a/htsget-config/README.md b/htsget-config/README.md
@@ -171,13 +171,14 @@ To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, and s
 `UrlStorage` is another storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication. 
 To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, and set the following options under `[resolvers.storage]`:
 
-| Option                                            | Description                                                                                                                                                      | Type       | Default                                                                                                   |
-|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-----------------------------------------------------------------------------------------------------------|
-| <span id="endpoint_index">`endpoint_index`</span> | The URL to fetch index for a file. The request will be a GET request which expects the index file specific to a BAM/CRAM/VCF file.                               | HTTP URL   | `"https://127.0.0.1:8081/"`                                                                               |
-| <span id="endpoint_header">`endpoint_file`</span> | The URL to fetch underlying for a file. The request will be a GET request which expects to get the decrypted underlying header from a BAM/CRAM/VCF file.         | HTTP URL   | `"https://127.0.0.1:8081/"`                                                                               |
-| <span id="url">`response_url`</span>              | The URL to return to the client for fetching tickets.                                                                                                            | HTTP URL   | `"https://127.0.0.1:8081/"`                                                                               |
-| `forward_headers`                                 | When constructing the URL tickets, copy HTTP headers received in the initial query. Note, the headers received with the query are always forwarded to the `url`. | Boolean    | `true`                                                                                                    |
-| `user_agent`                                      | A user agent to provide when making requests to the URLs.                                                                                                        | String     | A combination of the cargo package name and version. For example, `htsget-search/0.6.6`.                       |
+| Option                                            | Description                                                                                                                                                      | Type       | Default                                                                                                         |
+|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-----------------------------------------------------------------------------------------------------------------|
+| <span id="endpoint_index">`endpoint_index`</span> | The URL to fetch index for a file. The request will be a GET request which expects the index file specific to a BAM/CRAM/VCF file.                               | HTTP URL   | `"https://127.0.0.1:8081/"`                                                                                     |
+| <span id="endpoint_header">`endpoint_file`</span> | The URL to fetch underlying for a file. The request will be a GET request which expects to get the decrypted underlying header from a BAM/CRAM/VCF file.         | HTTP URL   | `"https://127.0.0.1:8081/"`                                                                                     |
+| <span id="url">`response_url`</span>              | The URL to return to the client for fetching tickets.                                                                                                            | HTTP URL   | `"https://127.0.0.1:8081/"`                                                                                     |
+| `forward_headers`                                 | When constructing the URL tickets, copy HTTP headers received in the initial query. Note, the headers received with the query are always forwarded to the `url`. | Boolean    | `true`                                                                                                          |
+| `user_agent`                                      | A user agent to provide when making requests to the URLs.                                                                                                        | String     | A combination of the cargo package name and version. For example, `htsget-search/0.6.6`.                        |
+| `danger_accept_invalid_certs`                     | Trusted invalid certificates, such as self-signed certificates. Only affects TLS on the HTTP client in `UrlStorage`.                                             | Boolean    | false                                                                                                           |
 | `tls`                                             | Additionally enables client authentication, or sets non-native root certificates for TLS. See [TLS](#tls) for more details.                                      | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. |
 
 When using `UrlStorage`, the following requests will be made:
@@ -278,11 +279,11 @@ TLS can be configured for the ticket server, data server, or the url storage cli
 certificates from PEM-formatted files. Certificates must be in X.509 format and private keys can be RSA, PKCS8, or SEC1 (EC) encoded. 
 The following options are available:
 
-| Option                 | Description                                                                                                                               | Type              | Default |
-|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------|
-| `key`                  | The path to the PEM formatted X.509 certificate. Specifies TLS for servers or client authentication for clients.                          | Filesystem path   | Not Set | 
-| `cert`                 | The path to the PEM formatted RSA, PKCS8, or SEC1 encoded EC private key. Specifies TLS for servers or client authentication for clients. | Filesystem path   | Not Set |
-| `root_store`           | The path to the PEM formatted root certificate store. Only used to specify non-native root certificates for client TLS.                   | Filesystem path   | Not Set |
+| Option                         | Description                                                                                                                               | Type            | Default |
+|--------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-----------------|---------|
+| `key`                          | The path to the PEM formatted X.509 certificate. Specifies TLS for servers or client authentication for clients.                          | Filesystem path | Not Set | 
+| `cert`                         | The path to the PEM formatted RSA, PKCS8, or SEC1 encoded EC private key. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set |
+| `root_store`                   | The path to the PEM formatted root certificate store. Only used to specify non-native root certificates for client TLS.                   | Filesystem path | Not Set |
 
 When used by the ticket and data servers, `key` and `cert` enable TLS, and when used with the url storage client, they enable client authentication.
 The root store is only used by the url storage client. Note, the url storage client always allows TLS, however the default configuration performs no client authentication

diff --git a/htsget-config/examples/config-files/crypt4gh.toml b/htsget-config/examples/config-files/crypt4gh.toml
@@ -1,25 +1,32 @@
 # An example that treats files as Crypt4GH encrypted.
 # Run with `cargo run -p htsget-actix --features crypt4gh,url-storage -- --config crypt4gh.toml`
 
-ticket_server_addr = "0.0.0.0:8080"
+ticket_server_addr = "0.0.0.0:7000"
 #data_server_addr = "0.0.0.0:8081"
 
 [[resolvers]]
 regex = ".*"
 substitution_string = "$0"
 
 [resolvers.object_type]
-# Specify the keys that htsget will use manually.
+# This option specified Crypt4GH files.
+send_encrypted_to_client = true
+# Specify the keys that htsget will use manually. These can be commented out to generate keys.
 private_key = "data/crypt4gh/keys/bob.sec" # pragma: allowlist secret
 public_key = "data/crypt4gh/keys/bob.pub"
 
-# Or, generate keys for each request.
-#object_type = "GenerateKeys"
-
 [resolvers.storage]
 response_url = "https://example.com/"
 forward_headers = false
 
+# Add a custom user agent.
+#user_agent = "user-agent"
+# Trust invalid certificates.
+#danger_accept_invalid_certs = true
+
+# Add a certificate to the client.
+#tls.root_store = "htsget-config/examples/config-files/cert.pem"
+
 [resolvers.storage.endpoints]
 file = "https://example.com/"
 index = "https://example.com/"
diff --git a/htsget-config/src/storage/url/mod.rs b/htsget-config/src/storage/url/mod.rs
@@ -4,6 +4,7 @@ use http::Uri as InnerUrl;
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
 use serde_with::with_prefix;
+use tracing::debug;
 
 use crate::error::Error::{ConfigError, ParseError};
 use crate::error::{Error, Result};
@@ -29,6 +30,7 @@ pub struct UrlStorage {
   response_url: ValidatedUrl,
   forward_headers: bool,
   user_agent: Option<String>,
+  danger_accept_invalid_certs: bool,
   #[serde(skip_serializing)]
   tls: TlsClientConfig,
 }
@@ -51,10 +53,13 @@ impl TryFrom<UrlStorage> for UrlStorageClient {
 
     let (_, cert, identity) = storage.tls.into_inner();
 
+    builder = builder.danger_accept_invalid_certs(storage.danger_accept_invalid_certs);
     if let Some(cert) = cert {
+      debug!("adding custom root certificate");
       builder = builder.add_root_certificate(cert);
     }
     if let Some(identity) = identity {
+      debug!("adding client authentication identity");
       builder = builder.identity(identity);
     }
 
@@ -160,6 +165,7 @@ impl UrlStorage {
     response_url: InnerUrl,
     forward_headers: bool,
     user_agent: Option<String>,
+    danger_accept_invalid_certs: bool,
     tls: TlsClientConfig,
   ) -> Self {
     Self {
@@ -169,6 +175,7 @@ impl UrlStorage {
       }),
       forward_headers,
       user_agent,
+      danger_accept_invalid_certs,
       tls,
     }
   }
@@ -207,6 +214,7 @@ impl Default for UrlStorage {
       response_url: default_url(),
       forward_headers: true,
       user_agent: None,
+      danger_accept_invalid_certs: false,
       tls: Default::default(),
     }
   }

diff --git a/htsget-config/src/tls/mod.rs b/htsget-config/src/tls/mod.rs
@@ -7,7 +7,7 @@ use rustls::{Certificate, ClientConfig, RootCertStore, ServerConfig};
 use rustls_native_certs::load_native_certs;
 use rustls_pemfile::read_one;
 use serde::{Deserialize, Serialize};
-use tracing::warn;
+use tracing::{debug, warn};
 
 use crate::error::Error::{IoError, ParseError};
 use crate::error::{Error, Result};
@@ -43,6 +43,8 @@ pub struct TlsClientConfig {
 
 impl Default for TlsClientConfig {
   fn default() -> Self {
+    debug!("using default client TLS config");
+
     Self {
       client_config: ClientConfig::builder()
         .with_safe_defaults()
@@ -306,7 +308,7 @@ pub fn load_reqwest_identity<P: AsRef<Path>>(key: P, cert: P) -> Result<reqwest:
   let key = read_bytes(key)?;
   let cert = read_bytes(cert)?;
 
-  reqwest::Identity::from_pkcs8_pem(&cert, &key)
+  reqwest::Identity::from_pem(&[cert, key].concat())
     .map_err(|err| IoError(format!("failed to pkcs8 pem identity: {}", err)))
 }
 

diff --git a/htsget-search/Cargo.toml b/htsget-search/Cargo.toml
@@ -79,7 +79,7 @@ tracing = "0.1"
 base64 = "0.21"
 serde = "1.0"
 
-reqwest = { version = "0.11", features = ["rustls-tls", "native-tls", "stream"], default-features = false }
+reqwest = { version = "0.11", features = ["rustls-tls", "stream"], default-features = false }
 
 [dev-dependencies]
 tempfile = "3.6"