From 90db32b66b2fa71c98ec292eb9f0ebf262749df9 Mon Sep 17 00:00:00 2001 From: Emil Ernerfeldt Date: Sun, 26 Feb 2023 17:20:18 +0100 Subject: [PATCH] Replace `reqwest` with `ureq` (#1407) * Replace reqwest with ureq for less dependencies * Simplify code * Cleanup --- Cargo.lock | 130 +++------------------ Cargo.toml | 1 - crates/re_analytics/Cargo.toml | 4 +- crates/re_analytics/src/pipeline_native.rs | 3 - crates/re_analytics/src/sink_native.rs | 107 ++++++++++------- crates/re_log/src/lib.rs | 2 +- deny.toml | 1 + examples/rust/raw_mesh/Cargo.toml | 1 - 8 files changed, 89 insertions(+), 160 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f34d2c1c65bb..108834828edf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -371,12 +371,6 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" -[[package]] -name = "base64" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" - [[package]] name = "bincode" version = "1.3.3" @@ -1415,15 +1409,6 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" -[[package]] -name = "encoding_rs" -version = "0.8.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" -dependencies = [ - "cfg-if", -] - [[package]] name = "enumn" version = "0.1.6" @@ -2208,19 +2193,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "rustls", - "tokio", - "tokio-rustls", -] - [[package]] name = "iana-time-zone" version = "0.1.53" @@ -2369,12 +2341,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "ipnet" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" - [[package]] name = "is-terminal" version = "0.4.2" @@ -2762,12 +2728,6 @@ dependencies = [ "libmimalloc-sys", ] -[[package]] -name = "mime" -version = "0.3.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" - [[package]] name = "minimal" version = "0.2.0" @@ -3825,7 +3785,6 @@ dependencies = [ "clap 4.1.4", "gltf", "mimalloc", - "reqwest", "rerun", ] @@ -3869,13 +3828,13 @@ dependencies = [ "re_build_build_info", "re_build_info", "re_log", - "reqwest", "serde", "serde_json", "sha2", "thiserror", "time 0.3.17", "tracing-subscriber", + "ureq", "uuid", "web-sys", ] @@ -4390,45 +4349,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1382d1f0a252c4bf97dc20d979a2fdd05b024acd7c2ed0f7595d7817666a157" -[[package]] -name = "reqwest" -version = "0.11.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" -dependencies = [ - "base64 0.21.0", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-rustls", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-rustls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "webpki-roots", - "winreg", -] - [[package]] name = "rerun" version = "0.2.0" @@ -4619,15 +4539,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" -dependencies = [ - "base64 0.21.0", -] - [[package]] name = "rustversion" version = "1.0.9" @@ -4771,18 +4682,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "sha-1" version = "0.10.0" @@ -5560,6 +5459,24 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" +dependencies = [ + "base64 0.13.1", + "flate2", + "log", + "once_cell", + "rustls", + "serde", + "serde_json", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "url" version = "2.3.1" @@ -6291,15 +6208,6 @@ dependencies = [ "x11-dl", ] -[[package]] -name = "winreg" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" -dependencies = [ - "winapi", -] - [[package]] name = "wit-parser" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index d3e0f6381f71..bac897a5c9e6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,6 @@ polars-core = "0.27.1" polars-lazy = "0.27.1" polars-ops = "0.27.1" puffin = "0.14" -reqwest = { version = "0.11", default-features = false } thiserror = "1.0" time = "0.3" tokio = "1.24" diff --git a/crates/re_analytics/Cargo.toml b/crates/re_analytics/Cargo.toml index e06d793bb660..2f2c776f2a94 100644 --- a/crates/re_analytics/Cargo.toml +++ b/crates/re_analytics/Cargo.toml @@ -34,7 +34,9 @@ uuid = { version = "1.1", features = ["serde", "v4", "js"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] directories-next = "2" -reqwest = { workspace = true, features = ["blocking", "rustls-tls"] } +ureq = { version = "2.6", features = [ + "json", +] } # TODO(emilk): use ehttp for web supprt [target.'cfg(target_arch = "wasm32")'.dependencies] web-sys = { version = "0.3.58", features = ["Storage"] } diff --git a/crates/re_analytics/src/pipeline_native.rs b/crates/re_analytics/src/pipeline_native.rs index 3b2920c9655b..969ea6a6634a 100644 --- a/crates/re_analytics/src/pipeline_native.rs +++ b/crates/re_analytics/src/pipeline_native.rs @@ -25,9 +25,6 @@ pub enum PipelineError { #[error(transparent)] Serde(#[from] serde_json::Error), - - #[error(transparent)] - Http(#[from] reqwest::Error), } /// An eventual, at-least-once(-ish) event pipeline, backed by a write-ahead log on the local disk. diff --git a/crates/re_analytics/src/sink_native.rs b/crates/re_analytics/src/sink_native.rs index 65c3f47dab19..c2bbe84fef24 100644 --- a/crates/re_analytics/src/sink_native.rs +++ b/crates/re_analytics/src/sink_native.rs @@ -1,11 +1,7 @@ -use std::{collections::HashMap, time::Duration}; +use std::collections::HashMap; -use once_cell::sync::OnceCell; -use reqwest::{blocking::Client as HttpClient, Url}; use time::OffsetDateTime; -use re_log::{debug, error}; - use crate::{Event, Property}; // TODO(cmc): abstract away the concept of a `Sink` behind an actual trait when comes the time to @@ -15,6 +11,9 @@ const PUBLIC_POSTHOG_PROJECT_KEY: &str = "phc_sgKidIE4WYYFSJHd8LEYY1UZqASpnfQKeM // --- +#[derive(Debug, Clone)] +struct Url(String); + #[derive(thiserror::Error, Debug)] pub enum SinkError { #[error(transparent)] @@ -24,19 +23,46 @@ pub enum SinkError { Serde(#[from] serde_json::Error), #[error(transparent)] - Http(#[from] reqwest::Error), + HttpTransport(Box), + + #[error("HTTP status {status_code} {status_text}: {body}")] + HttpStatus { + status_code: u16, + status_text: String, + body: String, + }, +} + +impl From for SinkError { + fn from(err: ureq::Error) -> Self { + match err { + ureq::Error::Status(status_code, response) => Self::HttpStatus { + status_code, + status_text: response.status_text().to_owned(), + body: response.into_string().unwrap_or_default(), + }, + + ureq::Error::Transport(transport) => Self::HttpTransport(Box::new(transport)), + } + } } -#[derive(Default, Debug, Clone)] +#[derive(Debug, Clone)] pub struct PostHogSink { - // NOTE: We need to lazily build the underlying HTTP client, so that we can guarantee that it - // is initialized from a thread that is free of a Tokio runtime. - // This is necessary because `reqwest` will crash if we try and initialize a blocking HTTP - // client from within a thread that has a Tokio runtime instantiated. - // - // We also use this opportunity to upgrade our public HTTP endpoint into the final HTTP2/TLS - // URL by following all 301 redirects. - client: OnceCell<(Url, HttpClient)>, + agent: ureq::Agent, + // Lazily resolve the url so that we don't do blocking requests in `PostHogSink::default` + resolved_url: once_cell::sync::OnceCell, +} + +impl Default for PostHogSink { + fn default() -> Self { + Self { + agent: ureq::AgentBuilder::new() + .timeout(std::time::Duration::from_secs(5)) + .build(), + resolved_url: Default::default(), + } + } } impl PostHogSink { @@ -50,7 +76,7 @@ impl PostHogSink { session_id: &str, events: &[Event], ) -> Result<(), SinkError> { - let (resolved_url, client) = self.init()?; + let resolved_url = self.init()?; let events = events .iter() @@ -58,34 +84,31 @@ impl PostHogSink { .collect::>(); let batch = PostHogBatch::from_events(&events); - debug!("{}", serde_json::to_string_pretty(&batch)?); - let resp = client - .post(resolved_url.clone()) - .body(serde_json::to_vec(&batch)?) - .send()?; - - resp.error_for_status().map(|_| ()).map_err(Into::into) + re_log::debug!("{}", serde_json::to_string_pretty(&batch)?); + self.agent.post(resolved_url).send_json(&batch)?; + Ok(()) } - fn init(&self) -> Result<&(Url, HttpClient), SinkError> { - self.client.get_or_try_init(|| { - use reqwest::header; - let mut headers = header::HeaderMap::new(); - headers.insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static("application/json"), - ); - - let client = HttpClient::builder() - .timeout(Duration::from_secs(5)) - .connect_timeout(Duration::from_secs(5)) - .pool_idle_timeout(Duration::from_secs(120)) - .default_headers(headers) - .build()?; - - let resolved_url = client.get(Self::URL).send()?.url().clone(); - - Ok((resolved_url, client)) + fn init(&self) -> Result<&String, SinkError> { + self.resolved_url.get_or_try_init(|| { + // Make a dummy-request to resolve our final URL. + let resolved_url = match self.agent.get(Self::URL).call() { + Ok(response) => response.get_url().to_owned(), + Err(ureq::Error::Status(status, response)) => { + // We actually expect to get here, because we make a bad request (GET to and end-point that expects a POST). + // We only do this requests to get redirected to the final URL. + let resolved_url = response.get_url().to_owned(); + re_log::trace!("status: {status} {}", response.status_text().to_owned()); + resolved_url + } + Err(ureq::Error::Transport(transport)) => { + return Err(SinkError::HttpTransport(Box::new(transport))) + } + }; + + // 2023-02-26 the resolved URL was https://eu.posthog.com/capture (in Europe) + + Ok(resolved_url) }) } } diff --git a/crates/re_log/src/lib.rs b/crates/re_log/src/lib.rs index aa8d4f9987fb..d9ba79e7ef11 100644 --- a/crates/re_log/src/lib.rs +++ b/crates/re_log/src/lib.rs @@ -31,8 +31,8 @@ fn set_default_rust_log_env() { // These are quite spammy on debug, drowning out what we care about: "h2", "hyper", - "reqwest", "rustls", + "ureq", ]; for loud_crate in LOUD_CRATES { if !rust_log.contains(&format!("{loud_crate}=")) { diff --git a/deny.toml b/deny.toml index 17620eb5e5f6..7586a305a4b6 100644 --- a/deny.toml +++ b/deny.toml @@ -35,6 +35,7 @@ deny = [ { name = "egui_glow" }, # We use wgpu { name = "openssl-sys" }, # We prefer rustls { name = "openssl" }, # We prefer rustls + { name = "reqwest" }, # We prefer ureq - less dependencies ] skip = [ diff --git a/examples/rust/raw_mesh/Cargo.toml b/examples/rust/raw_mesh/Cargo.toml index ca7672812b29..7f50ee7d1868 100644 --- a/examples/rust/raw_mesh/Cargo.toml +++ b/examples/rust/raw_mesh/Cargo.toml @@ -14,4 +14,3 @@ bytes = "1.3" clap = { workspace = true, features = ["derive"] } gltf.workspace = true mimalloc.workspace = true -reqwest = { workspace = true, features = ["blocking", "rustls-tls"] }