Switch from reqwest to bitreq HTTP client

.github/workflows/build.yml

@@ -6,11 +6,11 @@ jobs:
   build:
     strategy:
       matrix:
-        toolchain: [ stable, beta, 1.75.0 ] # 1.75.0 is current MSRV for vss-client-ng
+        toolchain: [ stable, beta, 1.85.0 ] # 1.85.0 is current MSRV for vss-client-ng
         include:
           - toolchain: stable
             check-fmt: true
-          - toolchain: 1.75.0
+          - toolchain: 1.85.0
             msrv: true
     runs-on: ubuntu-latest
     steps:
@@ -22,11 +22,6 @@ jobs:
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }}
           rustup override set ${{ matrix.toolchain }}
-      - name: Pin packages to allow for MSRV
-        if: matrix.msrv
-        run: |
-          cargo update -p idna_adapter --precise 1.1.0 # This has us use `unicode-normalization` which has a more conservative MSRV
-          cargo update -p proptest --precise "1.8.0" --verbose # proptest 1.9.0 requires rustc 1.82.0
       - name: Build on Rust ${{ matrix.toolchain }}
         run: cargo build --verbose --color always
       - name: Check formatting

Cargo.toml

@@ -2,7 +2,7 @@
 name = "vss-client-ng"
 version = "0.4.1"
 authors = ["Leo Nash <hello@leonash.net>", "Elias Rohrer <dev@tnull.de>"]
-rust-version = "1.75.0"
+rust-version = "1.85.0"
 license = "MIT OR Apache-2.0"
 edition = "2021"
 homepage = "https://lightningdevkit.org/"
@@ -15,28 +15,28 @@ build = "build.rs"
 
 [features]
 default = ["lnurl-auth", "sigs-auth"]
-lnurl-auth = ["dep:bitcoin", "dep:url", "dep:serde", "dep:serde_json", "reqwest/json"]
+lnurl-auth = ["dep:bitcoin", "dep:url", "dep:serde", "dep:serde_json"]
 sigs-auth = ["dep:bitcoin"]
 
 [dependencies]
 prost = "0.11.6"
-reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
+bitreq = { version = "0.3", default-features = false, features = ["async-https"] }
 tokio = { version = "1", default-features = false, features = ["time"] }
 rand = "0.8.5"
 async-trait = "0.1.77"
 bitcoin = { version = "0.32.2", default-features = false, features = ["std", "rand-std"], optional = true }
 url = { version = "2.5.0", default-features = false, optional = true }
-base64 = { version = "0.22", default-features = false}
+base64 = { version = "0.22", default-features = false, features = ["std"]}
 serde = { version = "1.0.196", default-features = false, features = ["serde_derive"], optional = true }
-serde_json = { version = "1.0.113", default-features = false, optional = true }
+serde_json = { version = "1.0.113", default-features = false, features = ["std"], optional = true }
 
 bitcoin_hashes = "0.14.0"
 chacha20-poly1305 = "0.1.2"
 log = { version = "0.4.29", default-features = false, features = ["std"]}
 
 [target.'cfg(genproto)'.build-dependencies]
 prost-build = { version = "0.11.3" }
-reqwest =  { version = "0.12", default-features = false, features = ["rustls-tls", "blocking"] }
+bitreq = { version = "0.3", default-features = false, features = ["std", "https"] }
 
 [dev-dependencies]
 mockito = "0.28.0"

README.md

@@ -9,4 +9,4 @@ and manage the essential state required for Lightning Network (LN) operations.
 Learn more [here](https://github.com/lightningdevkit/vss-server/blob/main/README.md).
 
 ## MSRV
-The Minimum Supported Rust Version (MSRV) is currently 1.75.0.
+The Minimum Supported Rust Version (MSRV) is currently 1.85.0.

build.rs

@@ -1,6 +1,8 @@
 #[cfg(genproto)]
 extern crate prost_build;
 #[cfg(genproto)]
+use std::io::Write;
+#[cfg(genproto)]
 use std::{env, fs, fs::File, path::Path};
 
 /// To generate updated proto objects:
@@ -14,7 +16,7 @@ fn main() {
 #[cfg(genproto)]
 fn generate_protos() {
 	download_file(
-				"https://raw.githubusercontent.com/lightningdevkit/vss-server/7f492fcac0c561b212f49ca40f7d16075822440f/app/src/main/proto/vss.proto",
+				"https://raw.githubusercontent.com/lightningdevkit/vss-server/022ee5e92debb60516438af0a369966495bfe595/proto/vss.proto",
 				"src/proto/vss.proto",
 		).unwrap();
 
@@ -25,9 +27,9 @@ fn generate_protos() {
 
 #[cfg(genproto)]
 fn download_file(url: &str, save_to: &str) -> Result<(), Box<dyn std::error::Error>> {
-	let mut response = reqwest::blocking::get(url)?;
+	let response = bitreq::get(url).send()?;
 	fs::create_dir_all(Path::new(save_to).parent().unwrap())?;
 	let mut out_file = File::create(save_to)?;
-	response.copy_to(&mut out_file)?;
+	out_file.write_all(&response.into_bytes())?;
 	Ok(())
 }

src/client.rs

@@ -1,14 +1,13 @@
+use bitreq::Client;
 use prost::Message;
-use reqwest::header::CONTENT_TYPE;
-use reqwest::Client;
 use std::collections::HashMap;
 use std::default::Default;
 use std::sync::Arc;
 
 use log::trace;
 
 use crate::error::VssError;
-use crate::headers::{get_headermap, FixedHeaders, VssHeaderProvider};
+use crate::headers::{FixedHeaders, VssHeaderProvider};
 use crate::types::{
 	DeleteObjectRequest, DeleteObjectResponse, GetObjectRequest, GetObjectResponse,
 	ListKeyVersionsRequest, ListKeyVersionsResponse, PutObjectRequest, PutObjectResponse,
@@ -17,7 +16,9 @@ use crate::util::retry::{retry, RetryPolicy};
 use crate::util::KeyValueVecKeyPrinter;
 
 const APPLICATION_OCTET_STREAM: &str = "application/octet-stream";
-const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
+const DEFAULT_TIMEOUT_SECS: u64 = 10;
+const MAX_RESPONSE_BODY_SIZE: usize = 500 * 1024 * 1024; // 500 MiB
+const DEFAULT_CLIENT_CAPACITY: usize = 10;
 
 /// Thin-client to access a hosted instance of Versioned Storage Service (VSS).
 /// The provided [`VssClient`] API is minimalistic and is congruent to the VSS server-side API.
@@ -35,11 +36,11 @@ where
 impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	/// Constructs a [`VssClient`] using `base_url` as the VSS server endpoint.
 	pub fn new(base_url: String, retry_policy: R) -> Self {
-		let client = build_client();
+		let client = Client::new(DEFAULT_CLIENT_CAPACITY);
 		Self::from_client(base_url, client, retry_policy)
 	}
 
-	/// Constructs a [`VssClient`] from a given [`reqwest::Client`], using `base_url` as the VSS server endpoint.
+	/// Constructs a [`VssClient`] from a given [`bitreq::Client`], using `base_url` as the VSS server endpoint.
 	pub fn from_client(base_url: String, client: Client, retry_policy: R) -> Self {
 		Self {
 			base_url,
@@ -49,7 +50,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		}
 	}
 
-	/// Constructs a [`VssClient`] from a given [`reqwest::Client`], using `base_url` as the VSS server endpoint.
+	/// Constructs a [`VssClient`] from a given [`bitreq::Client`], using `base_url` as the VSS server endpoint.
 	///
 	/// HTTP headers will be provided by the given `header_provider`.
 	pub fn from_client_and_headers(
@@ -65,7 +66,7 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 	pub fn new_with_headers(
 		base_url: String, retry_policy: R, header_provider: Arc<dyn VssHeaderProvider>,
 	) -> Self {
-		let client = build_client();
+		let client = Client::new(DEFAULT_CLIENT_CAPACITY);
 		Self { base_url, client, retry_policy, header_provider }
 	}
 
@@ -190,37 +191,30 @@ impl<R: RetryPolicy<E = VssError>> VssClient<R> {
 		&self, request: &Rq, url: &str,
 	) -> Result<Rs, VssError> {
 		let request_body = request.encode_to_vec();
-		let headermap = self
+		let headers = self
 			.header_provider
 			.get_headers(&request_body)
 			.await
-			.and_then(|h| get_headermap(&h))
 			.map_err(|e| VssError::AuthError(e.to_string()))?;
-		let response_raw = self
-			.client
-			.post(url)
-			.header(CONTENT_TYPE, APPLICATION_OCTET_STREAM)
-			.headers(headermap)
-			.body(request_body)
-			.send()
-			.await?;
-		let status = response_raw.status();
-		let payload = response_raw.bytes().await?;
-
-		if status.is_success() {
+
+		let http_request = bitreq::post(url)
+			.with_header("content-type", APPLICATION_OCTET_STREAM)
+			.with_headers(headers)
+			.with_body(request_body)
+			.with_timeout(DEFAULT_TIMEOUT_SECS)
+			.with_max_body_size(Some(MAX_RESPONSE_BODY_SIZE))
+			.with_pipelining();
+
+		let response = self.client.send_async(http_request).await?;
+
+		let status_code = response.status_code;
+		let payload = response.into_bytes();
+
+		if (200..300).contains(&status_code) {
 			let response = Rs::decode(&payload[..])?;
 			Ok(response)
 		} else {
-			Err(VssError::new(status, payload))
+			Err(VssError::new(status_code, payload))
 		}
 	}
 }
-
-fn build_client() -> Client {
-	Client::builder()
-		.timeout(DEFAULT_TIMEOUT)
-		.connect_timeout(DEFAULT_TIMEOUT)
-		.read_timeout(DEFAULT_TIMEOUT)
-		.build()
-		.unwrap()
-}

src/error.rs

@@ -1,7 +1,5 @@
 use crate::types::{ErrorCode, ErrorResponse};
-use prost::bytes::Bytes;
 use prost::{DecodeError, Message};
-use reqwest::StatusCode;
 use std::error::Error;
 use std::fmt::{Display, Formatter};
 
@@ -32,13 +30,13 @@ pub enum VssError {
 
 impl VssError {
 	/// Create new instance of `VssError`
-	pub fn new(status: StatusCode, payload: Bytes) -> VssError {
+	pub fn new(status_code: i32, payload: Vec<u8>) -> VssError {
 		match ErrorResponse::decode(&payload[..]) {
 			Ok(error_response) => VssError::from(error_response),
 			Err(e) => {
 				let message = format!(
 					"Unable to decode ErrorResponse from server, HttpStatusCode: {}, DecodeErr: {}",
-					status, e
+					status_code, e
 				);
 				VssError::InternalError(message)
 			},
@@ -99,8 +97,8 @@ impl From<DecodeError> for VssError {
 	}
 }
 
-impl From<reqwest::Error> for VssError {
-	fn from(err: reqwest::Error) -> Self {
+impl From<bitreq::Error> for VssError {
+	fn from(err: bitreq::Error) -> Self {
 		VssError::InternalError(err.to_string())
 	}
 }

src/headers/lnurl_auth_jwt.rs

@@ -1,4 +1,4 @@
-use crate::headers::{get_headermap, VssHeaderProvider, VssHeaderProviderError};
+use crate::headers::{VssHeaderProvider, VssHeaderProviderError};
 use async_trait::async_trait;
 use base64::engine::general_purpose::URL_SAFE_NO_PAD;
 use base64::Engine;
@@ -45,13 +45,14 @@ impl JwtToken {
 	}
 }
 
+const DEFAULT_TIMEOUT_SECS: u64 = 10;
+
 /// Provides a JWT token based on LNURL Auth.
 pub struct LnurlAuthToJwtProvider {
 	engine: Secp256k1<SignOnly>,
 	parent_key: Xpriv,
 	url: String,
 	default_headers: HashMap<String, String>,
-	client: reqwest::Client,
 	cached_jwt_token: RwLock<Option<JwtToken>>,
 }
 
@@ -70,47 +71,44 @@ impl LnurlAuthToJwtProvider {
 	/// with the JWT authorization header for VSS requests.
 	pub fn new(
 		parent_key: Xpriv, url: String, default_headers: HashMap<String, String>,
-	) -> Result<LnurlAuthToJwtProvider, VssHeaderProviderError> {
+	) -> LnurlAuthToJwtProvider {
 		let engine = Secp256k1::signing_only();
-		let default_headermap = get_headermap(&default_headers)?;
-		let client = reqwest::Client::builder()
-			.default_headers(default_headermap)
-			.build()
-			.map_err(VssHeaderProviderError::from)?;
 
-		Ok(LnurlAuthToJwtProvider {
+		LnurlAuthToJwtProvider {
 			engine,
 			parent_key,
 			url,
 			default_headers,
-			client,
 			cached_jwt_token: RwLock::new(None),
-		})
+		}
 	}
 
 	async fn fetch_jwt_token(&self) -> Result<JwtToken, VssHeaderProviderError> {
 		// Fetch the LNURL.
-		let lnurl_str = self
-			.client
-			.get(&self.url)
-			.send()
-			.await
-			.map_err(VssHeaderProviderError::from)?
-			.text()
-			.await
-			.map_err(VssHeaderProviderError::from)?;
+		let lnurl_request = bitreq::get(&self.url)
+			.with_headers(self.default_headers.clone())
+			.with_timeout(DEFAULT_TIMEOUT_SECS);
+		let lnurl_response =
+			lnurl_request.send_async().await.map_err(VssHeaderProviderError::from)?;
+		let lnurl_str = String::from_utf8(lnurl_response.into_bytes()).map_err(|e| {
+			VssHeaderProviderError::InvalidData {
+				error: format!("LNURL response is not valid UTF-8: {}", e),
+			}
+		})?;
 
 		// Sign the LNURL and perform the request.
 		let signed_lnurl = sign_lnurl(&self.engine, &self.parent_key, &lnurl_str)?;
-		let lnurl_auth_response: LnurlAuthResponse = self
-			.client
-			.get(&signed_lnurl)
-			.send()
-			.await
-			.map_err(VssHeaderProviderError::from)?
-			.json()
-			.await
-			.map_err(VssHeaderProviderError::from)?;
+		let auth_request = bitreq::get(&signed_lnurl)
+			.with_headers(self.default_headers.clone())
+			.with_timeout(DEFAULT_TIMEOUT_SECS);
+		let auth_response =
+			auth_request.send_async().await.map_err(VssHeaderProviderError::from)?;
+		let lnurl_auth_response: LnurlAuthResponse =
+			serde_json::from_slice(&auth_response.into_bytes()).map_err(|e| {
+				VssHeaderProviderError::InvalidData {
+					error: format!("Failed to parse LNURL Auth response as JSON: {}", e),
+				}
+			})?;
 
 		let untrusted_token = match lnurl_auth_response {
 			LnurlAuthResponse { token: Some(token), .. } => token,
@@ -256,8 +254,8 @@ impl From<bitcoin::bip32::Error> for VssHeaderProviderError {
 	}
 }
 
-impl From<reqwest::Error> for VssHeaderProviderError {
-	fn from(e: reqwest::Error) -> VssHeaderProviderError {
+impl From<bitreq::Error> for VssHeaderProviderError {
+	fn from(e: bitreq::Error) -> VssHeaderProviderError {
 		VssHeaderProviderError::RequestError { error: e.to_string() }
 	}
 }

src/headers/mod.rs

@@ -1,10 +1,8 @@
 use async_trait::async_trait;
-use reqwest::header::HeaderMap;
 use std::collections::HashMap;
 use std::error::Error;
 use std::fmt::Display;
 use std::fmt::Formatter;
-use std::str::FromStr;
 
 #[cfg(feature = "lnurl-auth")]
 mod lnurl_auth_jwt;
@@ -94,18 +92,3 @@ impl VssHeaderProvider for FixedHeaders {
 		Ok(self.headers.clone())
 	}
 }
-
-pub(crate) fn get_headermap(
-	headers: &HashMap<String, String>,
-) -> Result<HeaderMap, VssHeaderProviderError> {
-	let mut headermap = HeaderMap::new();
-	for (name, value) in headers {
-		headermap.insert(
-			reqwest::header::HeaderName::from_str(&name)
-				.map_err(|e| VssHeaderProviderError::InvalidData { error: e.to_string() })?,
-			reqwest::header::HeaderValue::from_str(&value)
-				.map_err(|e| VssHeaderProviderError::InvalidData { error: e.to_string() })?,
-		);
-	}
-	Ok(headermap)
-}

src/lib.rs

@@ -12,8 +12,8 @@
 #![deny(missing_docs)]
 
 // Crate re-exports
+pub use bitreq;
 pub use prost;
-pub use reqwest;
 
 /// Implements a thin-client ([`client::VssClient`]) to access a hosted instance of Versioned Storage Service (VSS).
 pub mod client;