Enable TLS native root toggling at runtime (#2362)

## Summary

It turns out that on macOS, reading the native certificates can add
hundreds of milliseconds to client initialization. This PR makes
`--native-tls` a command-line flag, to toggle (at runtime) the choice of
the `webpki` roots or the native system roots.

You can't accomplish this kind of configuration with the `reqwest`
builder API, so instead, I pulled out the heart of that logic from the
crate
(https://github.com/seanmonstar/reqwest/blob/e3192638518d577759dd89da489175b8f992b12f/src/async_impl/client.rs#L498),
and modified it to allow toggling a choice of root.

Note that there's an open PR for this in reqwest
(seanmonstar/reqwest#1848), along with an issue
(seanmonstar/reqwest#1843), which I may ping,
but it's been around for a while and I believe reqwest is focused on its
next major release.

Closes #2346.

Cargo.toml

@@ -76,7 +76,7 @@ rand = { version = "0.8.5" }
 rayon = { version = "1.8.0" }
 reflink-copy = { version = "0.1.15" }
 regex = { version = "1.10.2" }
-reqwest = { version = "0.11.23", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls-native-roots"] }
+reqwest = { version = "0.11.23", default-features = false, features = ["json", "gzip", "brotli", "stream", "rustls-tls", "rustls-tls-native-roots"] }
 reqwest-middleware = { version = "0.2.4" }
 reqwest-retry = { version = "0.3.0" }
 rkyv = { version = "0.7.43", features = ["strict", "validation"] }

README.md

@@ -425,13 +425,18 @@ In addition, uv respects the following environment variables:
 
 ## Custom CA Certificates
 
-uv supports custom CA certificates (such as those needed by corporate proxies) by utilizing the
-system's trust store. To ensure this works out of the box, ensure your certificates are added to the
-system's trust store.
+By default, uv loads certificates from the bundled `webpki-roots` crate. The `webpki-roots` are a
+reliable set of trust roots from Mozilla, and including them in uv improves portability and
+performance (especially on macOS, where reading the system trust store incurs a significant delay).
+
+However, in some cases, you may want to use the platform's native certificate store, especially if
+you're relying on a corporate trust root (e.g., for a mandatory proxy) that's included in your
+system's certificate store. To instruct uv to use the system's trust store, run uv with the
+`--native-tls` command-line flag.
 
 If a direct path to the certificate is required (e.g., in CI), set the `SSL_CERT_FILE` environment
-variable to the path of the certificate bundle, to instruct uv to use that file instead of the
-system's trust store.
+variable to the path of the certificate bundle (alongside the `--native-tls` flag), to instruct uv
+to use that file instead of the system's trust store.
 
 ## Acknowledgements
 

crates/uv-client/Cargo.toml

@@ -49,6 +49,11 @@ tracing = { workspace = true }
 url = { workspace = true }
 urlencoding = { workspace = true }
 
+# These must be kept in-sync with those used by `reqwest`.
+rustls = { version = "0.21.10" }
+rustls-native-certs = { version = "0.6.3" }
+webpki-roots = { version = "0.25.4" }
+
 [dev-dependencies]
 anyhow = { workspace = true }
 hyper = { version = "0.14.28", features = ["server", "http1"] }

crates/uv-client/src/lib.rs

@@ -16,3 +16,4 @@ mod middleware;
 mod registry_client;
 mod remote_metadata;
 mod rkyvutil;
+mod tls;

crates/uv-client/src/registry_client.rs

@@ -32,12 +32,14 @@ use crate::html::SimpleHtml;
 use crate::middleware::{NetrcMiddleware, OfflineMiddleware};
 use crate::remote_metadata::wheel_metadata_from_remote_zip;
 use crate::rkyvutil::OwnedArchive;
-use crate::{CachedClient, CachedClientError, Error, ErrorKind};
+use crate::tls::Roots;
+use crate::{tls, CachedClient, CachedClientError, Error, ErrorKind};
 
 /// A builder for an [`RegistryClient`].
 #[derive(Debug, Clone)]
 pub struct RegistryClientBuilder {
     index_urls: IndexUrls,
+    native_tls: bool,
     retries: u32,
     connectivity: Connectivity,
     cache: Cache,
@@ -48,6 +50,7 @@ impl RegistryClientBuilder {
     pub fn new(cache: Cache) -> Self {
         Self {
             index_urls: IndexUrls::default(),
+            native_tls: false,
             cache,
             connectivity: Connectivity::Online,
             retries: 3,
@@ -75,6 +78,12 @@ impl RegistryClientBuilder {
         self
     }
 
+    #[must_use]
+    pub fn native_tls(mut self, native_tls: bool) -> Self {
+        self.native_tls = native_tls;
+        self
+    }
+
     #[must_use]
     pub fn cache<T>(mut self, cache: Cache) -> Self {
         self.cache = cache;
@@ -110,11 +119,19 @@ impl RegistryClientBuilder {
 
         // Initialize the base client.
         let client = self.client.unwrap_or_else(|| {
-            // Disallow any connections.
+            // Load the TLS configuration.
+            let tls = tls::load(if self.native_tls {
+                Roots::Native
+            } else {
+                Roots::Webpki
+            })
+            .expect("Failed to load TLS configuration.");
+
             let client_core = ClientBuilder::new()
                 .user_agent(user_agent_string)
                 .pool_max_idle_per_host(20)
-                .timeout(std::time::Duration::from_secs(timeout));
+                .timeout(std::time::Duration::from_secs(timeout))
+                .use_preconfigured_tls(tls);
 
             client_core.build().expect("Failed to build HTTP client.")
         });

crates/uv-client/src/tls.rs

@@ -0,0 +1,102 @@
+use rustls::ClientConfig;
+use tracing::warn;
+
+#[derive(thiserror::Error, Debug)]
+pub(crate) enum TlsError {
+    #[error(transparent)]
+    Rustls(#[from] rustls::Error),
+    #[error("zero valid certificates found in native root store")]
+    ZeroCertificates,
+    #[error("failed to load native root certificates")]
+    NativeCertificates(#[source] std::io::Error),
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(crate) enum Roots {
+    /// Use reqwest's `rustls-tls-webpki-roots` behavior for loading root certificates.
+    Webpki,
+    /// Use reqwest's `rustls-tls-native-roots` behavior for loading root certificates.
+    Native,
+}
+
+/// Initialize a TLS configuration for the client.
+///
+/// This is equivalent to the TLS initialization `reqwest` when `rustls-tls` is enabled,
+/// with two notable changes:
+///
+/// 1. It enables _either_ the `webpki-roots` or the `native-certs` feature, but not both.
+/// 2. It assumes the following builder settings (which match the defaults):
+///    - `root_certs: vec![]`
+///    - `min_tls_version: None`
+///    - `max_tls_version: None`
+///    - `identity: None`
+///    - `certs_verification: false`
+///    - `tls_sni: true`
+///    - `http_version_pref: HttpVersionPref::All`
+///
+/// See: <https://github.com/seanmonstar/reqwest/blob/e3192638518d577759dd89da489175b8f992b12f/src/async_impl/client.rs#L498>
+pub(crate) fn load(roots: Roots) -> Result<ClientConfig, TlsError> {
+    // Set root certificates.
+    let mut root_cert_store = rustls::RootCertStore::empty();
+
+    match roots {
+        Roots::Webpki => {
+            // Use `rustls-tls-webpki-roots`
+            use rustls::OwnedTrustAnchor;
+
+            let trust_anchors = webpki_roots::TLS_SERVER_ROOTS.iter().map(|trust_anchor| {
+                OwnedTrustAnchor::from_subject_spki_name_constraints(
+                    trust_anchor.subject,
+                    trust_anchor.spki,
+                    trust_anchor.name_constraints,
+                )
+            });
+
+            root_cert_store.add_trust_anchors(trust_anchors);
+        }
+        Roots::Native => {
+            // Use: `rustls-tls-native-roots`
+            let mut valid_count = 0;
+            let mut invalid_count = 0;
+            for cert in
+                rustls_native_certs::load_native_certs().map_err(TlsError::NativeCertificates)?
+            {
+                let cert = rustls::Certificate(cert.0);
+                // Continue on parsing errors, as native stores often include ancient or syntactically
+                // invalid certificates, like root certificates without any X509 extensions.
+                // Inspiration: https://github.com/rustls/rustls/blob/633bf4ba9d9521a95f68766d04c22e2b01e68318/rustls/src/anchors.rs#L105-L112
+                match root_cert_store.add(&cert) {
+                    Ok(_) => valid_count += 1,
+                    Err(err) => {
+                        invalid_count += 1;
+                        warn!(
+                            "rustls failed to parse DER certificate {:?} {:?}",
+                            &err, &cert
+                        );
+                    }
+                }
+            }
+            if valid_count == 0 && invalid_count > 0 {
+                return Err(TlsError::ZeroCertificates);
+            }
+        }
+    }
+
+    // Build TLS config
+    let config_builder = ClientConfig::builder()
+        .with_safe_default_cipher_suites()
+        .with_safe_default_kx_groups()
+        .with_protocol_versions(rustls::ALL_VERSIONS)?
+        .with_root_certificates(root_cert_store);
+
+    // Finalize TLS config
+    let mut tls = config_builder.with_no_client_auth();
+
+    // Enable SNI
+    tls.enable_sni = true;
+
+    // ALPN protocol
+    tls.alpn_protocols = vec!["h2".into(), "http/1.1".into()];
+
+    Ok(tls)
+}

crates/uv/src/commands/pip_compile.rs

@@ -67,6 +67,7 @@ pub(crate) async fn pip_compile(
     python_version: Option<PythonVersion>,
     exclude_newer: Option<DateTime<Utc>>,
     annotation_style: AnnotationStyle,
+    native_tls: bool,
     quiet: bool,
     cache: Cache,
     printer: Printer,
@@ -188,6 +189,7 @@ pub(crate) async fn pip_compile(
 
     // Initialize the registry client.
     let client = RegistryClientBuilder::new(cache.clone())
+        .native_tls(native_tls)
         .connectivity(connectivity)
         .index_urls(index_locations.index_urls())
         .build();

crates/uv/src/commands/pip_install.rs

@@ -67,6 +67,7 @@ pub(crate) async fn pip_install(
     python: Option<String>,
     system: bool,
     break_system_packages: bool,
+    native_tls: bool,
     cache: Cache,
     printer: Printer,
 ) -> Result<ExitStatus> {
@@ -177,6 +178,7 @@ pub(crate) async fn pip_install(
 
     // Initialize the registry client.
     let client = RegistryClientBuilder::new(cache.clone())
+        .native_tls(native_tls)
         .connectivity(connectivity)
         .index_urls(index_locations.index_urls())
         .build();

crates/uv/src/commands/pip_sync.rs

@@ -45,6 +45,7 @@ pub(crate) async fn pip_sync(
     python: Option<String>,
     system: bool,
     break_system_packages: bool,
+    native_tls: bool,
     cache: Cache,
     printer: Printer,
 ) -> Result<ExitStatus> {
@@ -116,6 +117,7 @@ pub(crate) async fn pip_sync(
 
     // Initialize the registry client.
     let client = RegistryClientBuilder::new(cache.clone())
+        .native_tls(native_tls)
         .connectivity(connectivity)
         .index_urls(index_locations.index_urls())
         .build();

crates/uv/src/main.rs

@@ -88,6 +88,18 @@ struct Cli {
     )]
     color: ColorChoice,
 
+    /// Whether to load TLS certificates from the platform's native certificate store.
+    ///
+    /// By default, `uv` loads certificates from the bundled `webpki-roots` crate. The
+    /// `webpki-roots` are a reliable set of trust roots from Mozilla, and including them in `uv`
+    /// improves portability and performance (especially on macOS).
+    ///
+    /// However, in some cases, you may want to use the platform's native certificate store,
+    /// especially if you're relying on a corporate trust root (e.g., for a mandatory proxy) that's
+    /// included in your system's certificate store.
+    #[arg(global = true, long)]
+    native_tls: bool,
+
     #[command(flatten)]
     cache_args: CacheArgs,
 }
@@ -1419,6 +1431,7 @@ async fn run() -> Result<ExitStatus> {
                 args.python_version,
                 args.exclude_newer,
                 args.annotation_style,
+                cli.native_tls,
                 cli.quiet,
                 cache,
                 printer,
@@ -1475,6 +1488,7 @@ async fn run() -> Result<ExitStatus> {
                 args.python,
                 args.system,
                 args.break_system_packages,
+                cli.native_tls,
                 cache,
                 printer,
             )
@@ -1570,6 +1584,7 @@ async fn run() -> Result<ExitStatus> {
                 args.python,
                 args.system,
                 args.break_system_packages,
+                cli.native_tls,
                 cache,
                 printer,
             )