diff --git a/Cargo.lock b/Cargo.lock index 07b00f8c..c87b6140 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1759,21 +1759,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.1" @@ -2036,6 +2021,7 @@ dependencies = [ "http-serde", "hyper", "hyper-rustls", + "log", "noodles 0.65.0", "rcgen", "regex", @@ -2264,19 +2250,6 @@ dependencies = [ "tokio-rustls", ] -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2730,24 +2703,6 @@ version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d02c0b00610773bb7fc61d85e13d86c7858cbdf00e1a120bfc41bc055dbaa0e" -[[package]] -name = "native-tls" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" -dependencies = [ - "lazy_static", - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nom" version = "7.1.3" @@ -3196,50 +3151,12 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "openssl" -version = "0.10.64" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" -dependencies = [ - "bitflags 2.4.2", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.52", -] - [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" -[[package]] -name = "openssl-sys" -version = "0.9.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "outref" version = "0.5.1" @@ -3710,12 +3627,10 @@ dependencies = [ "http-body", "hyper", "hyper-rustls", - "hyper-tls", "ipnet", "js-sys", "log", "mime", - "native-tls", "once_cell", "percent-encoding", "pin-project-lite", @@ -3727,7 +3642,6 @@ dependencies = [ "sync_wrapper", "system-configuration", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower-service", @@ -4464,16 +4378,6 @@ dependencies = [ "syn 2.0.52", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -4815,12 +4719,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" diff --git a/htsget-config/Cargo.toml b/htsget-config/Cargo.toml index 91185efb..bbd72a58 100644 --- a/htsget-config/Cargo.toml +++ b/htsget-config/Cargo.toml @@ -37,10 +37,11 @@ rustls-native-certs = "0.6" hyper-rustls = { version = "0.24", features = ["rustls-native-certs", "http2", "http1"] } hyper = { version = "0.14", features = ["http1", "http2", "client"], optional = true } -reqwest = { version = "0.11", features = ["rustls-tls", "native-tls"], default-features = false } +reqwest = { version = "0.11", features = ["rustls-tls"], default-features = false } async-crypt4gh = { version = "0.1.0", path = "../async-crypt4gh", default-features = false, optional = true } crypt4gh = { version = "0.4", git = "https://github.com/EGA-archive/crypt4gh-rust", optional = true } +log = "0.4.21" [dev-dependencies] serde_json = "1.0" diff --git a/htsget-config/README.md b/htsget-config/README.md index 83bd5db1..2b296b54 100644 --- a/htsget-config/README.md +++ b/htsget-config/README.md @@ -171,13 +171,14 @@ To use `S3Storage`, build htsget-rs with the `s3-storage` feature enabled, and s `UrlStorage` is another storage backend which can be used to serve data from a remote HTTP URL. When using this storage backend, htsget-rs will fetch data from a `url` which is set in the config. It will also forward any headers received with the initial query, which is useful for authentication. To use `UrlStorage`, build htsget-rs with the `url-storage` feature enabled, and set the following options under `[resolvers.storage]`: -| Option | Description | Type | Default | -|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-----------------------------------------------------------------------------------------------------------| -| `endpoint_index` | The URL to fetch index for a file. The request will be a GET request which expects the index file specific to a BAM/CRAM/VCF file. | HTTP URL | `"https://127.0.0.1:8081/"` | -| `endpoint_file` | The URL to fetch underlying for a file. The request will be a GET request which expects to get the decrypted underlying header from a BAM/CRAM/VCF file. | HTTP URL | `"https://127.0.0.1:8081/"` | -| `response_url` | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` | -| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. Note, the headers received with the query are always forwarded to the `url`. | Boolean | `true` | -| `user_agent` | A user agent to provide when making requests to the URLs. | String | A combination of the cargo package name and version. For example, `htsget-search/0.6.6`. | +| Option | Description | Type | Default | +|---------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------|-----------------------------------------------------------------------------------------------------------------| +| `endpoint_index` | The URL to fetch index for a file. The request will be a GET request which expects the index file specific to a BAM/CRAM/VCF file. | HTTP URL | `"https://127.0.0.1:8081/"` | +| `endpoint_file` | The URL to fetch underlying for a file. The request will be a GET request which expects to get the decrypted underlying header from a BAM/CRAM/VCF file. | HTTP URL | `"https://127.0.0.1:8081/"` | +| `response_url` | The URL to return to the client for fetching tickets. | HTTP URL | `"https://127.0.0.1:8081/"` | +| `forward_headers` | When constructing the URL tickets, copy HTTP headers received in the initial query. Note, the headers received with the query are always forwarded to the `url`. | Boolean | `true` | +| `user_agent` | A user agent to provide when making requests to the URLs. | String | A combination of the cargo package name and version. For example, `htsget-search/0.6.6`. | +| `danger_accept_invalid_certs` | Trusted invalid certificates, such as self-signed certificates. Only affects TLS on the HTTP client in `UrlStorage`. | Boolean | false | | `tls` | Additionally enables client authentication, or sets non-native root certificates for TLS. See [TLS](#tls) for more details. | TOML table | TLS is always allowed, however the default performs no client authentication and uses native root certificates. | When using `UrlStorage`, the following requests will be made: @@ -278,11 +279,11 @@ TLS can be configured for the ticket server, data server, or the url storage cli certificates from PEM-formatted files. Certificates must be in X.509 format and private keys can be RSA, PKCS8, or SEC1 (EC) encoded. The following options are available: -| Option | Description | Type | Default | -|------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-------------------|---------| -| `key` | The path to the PEM formatted X.509 certificate. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | -| `cert` | The path to the PEM formatted RSA, PKCS8, or SEC1 encoded EC private key. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | -| `root_store` | The path to the PEM formatted root certificate store. Only used to specify non-native root certificates for client TLS. | Filesystem path | Not Set | +| Option | Description | Type | Default | +|--------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------|-----------------|---------| +| `key` | The path to the PEM formatted X.509 certificate. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | +| `cert` | The path to the PEM formatted RSA, PKCS8, or SEC1 encoded EC private key. Specifies TLS for servers or client authentication for clients. | Filesystem path | Not Set | +| `root_store` | The path to the PEM formatted root certificate store. Only used to specify non-native root certificates for client TLS. | Filesystem path | Not Set | When used by the ticket and data servers, `key` and `cert` enable TLS, and when used with the url storage client, they enable client authentication. The root store is only used by the url storage client. Note, the url storage client always allows TLS, however the default configuration performs no client authentication diff --git a/htsget-config/examples/config-files/crypt4gh.toml b/htsget-config/examples/config-files/crypt4gh.toml index b3fd8277..9c7602fd 100644 --- a/htsget-config/examples/config-files/crypt4gh.toml +++ b/htsget-config/examples/config-files/crypt4gh.toml @@ -1,7 +1,7 @@ # An example that treats files as Crypt4GH encrypted. # Run with `cargo run -p htsget-actix --features crypt4gh,url-storage -- --config crypt4gh.toml` -ticket_server_addr = "0.0.0.0:8080" +ticket_server_addr = "0.0.0.0:7000" #data_server_addr = "0.0.0.0:8081" [[resolvers]] @@ -9,17 +9,24 @@ regex = ".*" substitution_string = "$0" [resolvers.object_type] -# Specify the keys that htsget will use manually. +# This option specified Crypt4GH files. +send_encrypted_to_client = true +# Specify the keys that htsget will use manually. These can be commented out to generate keys. private_key = "data/crypt4gh/keys/bob.sec" # pragma: allowlist secret public_key = "data/crypt4gh/keys/bob.pub" -# Or, generate keys for each request. -#object_type = "GenerateKeys" - [resolvers.storage] response_url = "https://example.com/" forward_headers = false +# Add a custom user agent. +#user_agent = "user-agent" +# Trust invalid certificates. +#danger_accept_invalid_certs = true + +# Add a certificate to the client. +#tls.root_store = "htsget-config/examples/config-files/cert.pem" + [resolvers.storage.endpoints] file = "https://example.com/" index = "https://example.com/" \ No newline at end of file diff --git a/htsget-config/src/storage/url/mod.rs b/htsget-config/src/storage/url/mod.rs index daa19472..33b0ce54 100644 --- a/htsget-config/src/storage/url/mod.rs +++ b/htsget-config/src/storage/url/mod.rs @@ -4,6 +4,7 @@ use http::Uri as InnerUrl; use reqwest::Client; use serde::{Deserialize, Serialize}; use serde_with::with_prefix; +use tracing::debug; use crate::error::Error::{ConfigError, ParseError}; use crate::error::{Error, Result}; @@ -29,6 +30,7 @@ pub struct UrlStorage { response_url: ValidatedUrl, forward_headers: bool, user_agent: Option, + danger_accept_invalid_certs: bool, #[serde(skip_serializing)] tls: TlsClientConfig, } @@ -51,10 +53,13 @@ impl TryFrom for UrlStorageClient { let (_, cert, identity) = storage.tls.into_inner(); + builder = builder.danger_accept_invalid_certs(storage.danger_accept_invalid_certs); if let Some(cert) = cert { + debug!("adding custom root certificate"); builder = builder.add_root_certificate(cert); } if let Some(identity) = identity { + debug!("adding client authentication identity"); builder = builder.identity(identity); } @@ -160,6 +165,7 @@ impl UrlStorage { response_url: InnerUrl, forward_headers: bool, user_agent: Option, + danger_accept_invalid_certs: bool, tls: TlsClientConfig, ) -> Self { Self { @@ -169,6 +175,7 @@ impl UrlStorage { }), forward_headers, user_agent, + danger_accept_invalid_certs, tls, } } @@ -207,6 +214,7 @@ impl Default for UrlStorage { response_url: default_url(), forward_headers: true, user_agent: None, + danger_accept_invalid_certs: false, tls: Default::default(), } } diff --git a/htsget-config/src/tls/mod.rs b/htsget-config/src/tls/mod.rs index bcd72dfa..362d3570 100644 --- a/htsget-config/src/tls/mod.rs +++ b/htsget-config/src/tls/mod.rs @@ -7,7 +7,7 @@ use rustls::{Certificate, ClientConfig, RootCertStore, ServerConfig}; use rustls_native_certs::load_native_certs; use rustls_pemfile::read_one; use serde::{Deserialize, Serialize}; -use tracing::warn; +use tracing::{debug, warn}; use crate::error::Error::{IoError, ParseError}; use crate::error::{Error, Result}; @@ -43,6 +43,8 @@ pub struct TlsClientConfig { impl Default for TlsClientConfig { fn default() -> Self { + debug!("using default client TLS config"); + Self { client_config: ClientConfig::builder() .with_safe_defaults() @@ -306,7 +308,7 @@ pub fn load_reqwest_identity>(key: P, cert: P) -> Result