Merge pull request #170 from greatest-ape/ws-mem-leak

Possible fix for ws memory leak; dependency updates; add ws mimalloc feature

TODO.md

@@ -2,6 +2,9 @@
 
 ## High priority
 
+* if peer_clients is on, add task to generate prometheus exports on regular
+  interval to clean up data
+
 * aquatic_bench
   * Opentracker "slow to get up to speed", is it due to getting faster once
     inserts are rarely needed since most ip-port combinations have been sent?

crates/common/Cargo.toml

@@ -37,5 +37,5 @@ toml = "0.5"
 # Optional
 glommio = { version = "0.8", optional = true }
 hwloc = { version = "0.5", optional = true }
-rustls = { version = "0.21", optional = true }
-rustls-pemfile = { version = "1", optional = true }
+rustls = { version = "0.22", optional = true }
+rustls-pemfile = { version = "2", optional = true }

crates/common/src/rustls_config.rs

@@ -17,10 +17,20 @@ pub fn create_rustls_config(
         })?;
         let mut f = BufReader::new(f);
 
-        rustls_pemfile::certs(&mut f)?
-            .into_iter()
-            .map(|bytes| rustls::Certificate(bytes))
-            .collect()
+        let mut certs = Vec::new();
+
+        for cert in rustls_pemfile::certs(&mut f) {
+            match cert {
+                Ok(cert) => {
+                    certs.push(cert);
+                }
+                Err(err) => {
+                    ::log::error!("error parsing certificate: {:#?}", err)
+                }
+            }
+        }
+
+        certs
     };
 
     let private_key = {
@@ -32,16 +42,16 @@ pub fn create_rustls_config(
         })?;
         let mut f = BufReader::new(f);
 
-        rustls_pemfile::pkcs8_private_keys(&mut f)?
-            .first()
-            .map(|bytes| rustls::PrivateKey(bytes.clone()))
-            .ok_or(anyhow::anyhow!("No private keys in file"))?
+        let key = rustls_pemfile::pkcs8_private_keys(&mut f)
+            .next()
+            .ok_or(anyhow::anyhow!("No private keys in file"))??;
+
+        key
     };
 
     let tls_config = rustls::ServerConfig::builder()
-        .with_safe_defaults()
         .with_no_client_auth()
-        .with_single_cert(certs, private_key)
+        .with_single_cert(certs, rustls::pki_types::PrivateKeyDer::Pkcs8(private_key))
         .with_context(|| "create rustls config")?;
 
     Ok(tls_config)

crates/http/Cargo.toml

@@ -33,7 +33,7 @@ cfg-if = "1"
 either = "1"
 futures = "0.3"
 futures-lite = "1"
-futures-rustls = "0.24"
+futures-rustls = "0.25"
 glommio = "0.8"
 httparse = "1"
 itoa = "1"
@@ -46,7 +46,7 @@ memchr = "2"
 privdrop = "0.5"
 once_cell = "1"
 rand = { version = "0.8", features = ["small_rng"] }
-rustls-pemfile = "1"
+rustls-pemfile = "2"
 serde = { version = "1", features = ["derive"] }
 signal-hook = { version = "0.3" }
 slotmap = "1"

crates/http_load_test/Cargo.toml

@@ -21,14 +21,14 @@ aquatic_toml_config.workspace = true
 anyhow = "1"
 futures = "0.3"
 futures-lite = "1"
-futures-rustls = "0.24"
+futures-rustls = "0.25"
 hashbrown = "0.14"
 glommio = "0.8"
 log = "0.4"
 mimalloc = { version = "0.1", default-features = false }
 rand = { version = "0.8", features = ["small_rng"] }
 rand_distr = "0.4"
-rustls = { version = "0.21", default-features = false, features = ["logging", "dangerous_configuration"] } # TLS 1.2 disabled
+rustls = { version = "0.22", default-features = false, features = ["logging"] } # TLS 1.2 disabled
 serde = { version = "1", features = ["derive"] }
 
 [dev-dependencies]

crates/http_load_test/src/common.rs

@@ -4,7 +4,6 @@ use rand_distr::Gamma;
 
 pub use aquatic_http_protocol::common::*;
 pub use aquatic_http_protocol::request::*;
-pub use aquatic_http_protocol::response::*;
 
 #[derive(PartialEq, Eq, Clone)]
 pub struct TorrentPeer {

crates/http_load_test/src/main.rs

@@ -180,25 +180,53 @@ fn monitor_statistics(state: LoadTestState, config: &Config) {
     }
 }
 
+#[derive(Debug)]
 struct FakeCertificateVerifier;
 
-impl rustls::client::ServerCertVerifier for FakeCertificateVerifier {
+impl rustls::client::danger::ServerCertVerifier for FakeCertificateVerifier {
     fn verify_server_cert(
         &self,
-        _end_entity: &rustls::Certificate,
-        _intermediates: &[rustls::Certificate],
-        _server_name: &rustls::ServerName,
-        _scts: &mut dyn Iterator<Item = &[u8]>,
+        _end_entity: &rustls::pki_types::CertificateDer<'_>,
+        _intermediates: &[rustls::pki_types::CertificateDer<'_>],
+        _server_name: &rustls::pki_types::ServerName<'_>,
         _ocsp_response: &[u8],
-        _now: std::time::SystemTime,
-    ) -> Result<rustls::client::ServerCertVerified, rustls::Error> {
-        Ok(rustls::client::ServerCertVerified::assertion())
+        _now: rustls::pki_types::UnixTime,
+    ) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
+        Ok(rustls::client::danger::ServerCertVerified::assertion())
+    }
+
+    fn verify_tls12_signature(
+        &self,
+        _message: &[u8],
+        _cert: &rustls::pki_types::CertificateDer<'_>,
+        _dss: &rustls::DigitallySignedStruct,
+    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
+        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
+    }
+
+    fn verify_tls13_signature(
+        &self,
+        _message: &[u8],
+        _cert: &rustls::pki_types::CertificateDer<'_>,
+        _dss: &rustls::DigitallySignedStruct,
+    ) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
+        Ok(rustls::client::danger::HandshakeSignatureValid::assertion())
+    }
+
+    fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
+        vec![
+            rustls::SignatureScheme::ECDSA_NISTP384_SHA384,
+            rustls::SignatureScheme::ECDSA_NISTP256_SHA256,
+            rustls::SignatureScheme::RSA_PSS_SHA512,
+            rustls::SignatureScheme::RSA_PSS_SHA384,
+            rustls::SignatureScheme::RSA_PSS_SHA256,
+            rustls::SignatureScheme::ED25519,
+        ]
     }
 }
 
 fn create_tls_config() -> anyhow::Result<Arc<rustls::ClientConfig>> {
     let mut config = rustls::ClientConfig::builder()
-        .with_safe_defaults()
         .with_root_certificates(rustls::RootCertStore::empty())
         .with_no_client_auth();
 

crates/ws/Cargo.toml

@@ -18,9 +18,12 @@ name = "aquatic_ws"
 name = "aquatic_ws"
 
 [features]
-default = ["prometheus"]
+default = ["prometheus", "mimalloc"]
 prometheus = ["metrics", "metrics-exporter-prometheus"]
 metrics = ["dep:metrics", "metrics-util"]
+# Use mimalloc allocator for much better performance. Requires cmake and a
+# C/C++ compiler
+mimalloc = ["dep:mimalloc"]
 
 [dependencies]
 aquatic_common = { workspace = true, features = ["rustls", "glommio"] }
@@ -29,31 +32,31 @@ aquatic_toml_config.workspace = true
 aquatic_ws_protocol.workspace = true
 
 anyhow = "1"
-async-tungstenite = "0.23"
+async-tungstenite = "0.24"
 arc-swap = "1"
 cfg-if = "1"
 futures = "0.3"
 futures-lite = "1"
-futures-rustls = "0.24"
+futures-rustls = "0.25"
 glommio = "0.8"
 hashbrown = { version = "0.14", features = ["serde"] }
 httparse = "1"
 indexmap = "2"
 log = "0.4"
-metrics = { version = "0.21", optional = true }
-metrics-util = { version = "0.15", optional = true }
-metrics-exporter-prometheus = { version = "0.12", optional = true, default-features = false, features = ["http-listener"] }
-mimalloc = { version = "0.1", default-features = false }
+metrics = { version = "0.22", optional = true }
+metrics-util = { version = "0.16", optional = true }
+metrics-exporter-prometheus = { version = "0.13", optional = true, default-features = false, features = ["http-listener"] }
+mimalloc = { version = "0.1", default-features = false, optional = true }
 privdrop = "0.5"
 rand = { version = "0.8", features = ["small_rng"] }
-rustls = "0.21"
-rustls-pemfile = "1"
+rustls = "0.22"
+rustls-pemfile = "2"
 serde = { version = "1", features = ["derive"] }
 signal-hook = { version = "0.3" }
 slab = "0.4"
 slotmap = "1"
 socket2 = { version = "0.5", features = ["all"] }
-tungstenite = "0.20"
+tungstenite = "0.21"
 
 [dev-dependencies]
 quickcheck = "1"

crates/ws/src/main.rs

@@ -1,6 +1,7 @@
 use aquatic_common::cli::run_app_with_cli_and_config;
 use aquatic_ws::config::Config;
 
+#[cfg(feature = "mimalloc")]
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;