refactor(bench): separate out client and server connections in benchi…

…ng harness (#4113)

Enables more better control of connections for benching experiments

bindings/rust/bench/Cargo.toml

@@ -10,7 +10,7 @@ historical-perf = []
 s2n-tls = { path = "../s2n-tls" }
 rustls = "0.21"
 rustls-pemfile = "1.0"
-openssl = "0.10"
+openssl = { version = "0.10", features = ["vendored"] }
 errno = "0.3"
 libc = "0.2"
 crabgrind = "0.1"
@@ -19,9 +19,10 @@ rand_distr = "0.4"
 plotters = "0.3"
 serde_json = "1.0"
 semver = "1.0"
+strum = { version = "0.25", features = ["derive"] }
 
 [dev-dependencies]
-criterion = "0.3"
+criterion = "0.5"
 
 [[bench]]
 name = "handshake"

bindings/rust/bench/benches/handshake.rs

@@ -2,77 +2,72 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use bench::{
-    CipherSuite, CryptoConfig,
-    ECGroup::{self, *},
-    HandshakeType::{self, *},
-    OpenSslHarness, RustlsHarness, S2NHarness,
-    SigType::{self, *},
-    TlsBenchHarness,
+    CipherSuite, CryptoConfig, HandshakeType, KXGroup, OpenSslConnection, RustlsConnection,
+    S2NConnection, SigType, TlsConnPair, TlsConnection,
 };
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion,
 };
+use strum::IntoEnumIterator;
+
+fn bench_handshake_for_library<T: TlsConnection>(
+    bench_group: &mut BenchmarkGroup<WallTime>,
+    handshake_type: HandshakeType,
+    kx_group: KXGroup,
+    sig_type: SigType,
+) {
+    // generate all harnesses (TlsConnPair structs) beforehand so that benchmarks
+    // only include negotiation and not config/connection initialization
+    bench_group.bench_function(T::name(), |b| {
+        b.iter_batched_ref(
+            || {
+                TlsConnPair::<T, T>::new(
+                    CryptoConfig::new(CipherSuite::default(), kx_group, sig_type),
+                    handshake_type,
+                    Default::default(),
+                )
+            },
+            |conn_pair_res| {
+                // harnesses with certain parameters fail to initialize for
+                // some past versions of s2n-tls, but missing data can be
+                // visually interpolated in the historical performance graph
+                if let Ok(conn_pair) = conn_pair_res {
+                    let _ = conn_pair.handshake();
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    });
+}
 
 pub fn bench_handshake_params(c: &mut Criterion) {
-    fn bench_handshake_for_library<T: TlsBenchHarness>(
-        bench_group: &mut BenchmarkGroup<WallTime>,
-        name: &str,
-        handshake_type: HandshakeType,
-        ec_group: ECGroup,
-        sig_type: SigType,
-    ) {
-        // generate all harnesses (TlsBenchHarness structs) beforehand so that benchmarks
-        // only include negotiation and not config/connection initialization
-        bench_group.bench_function(name, |b| {
-            b.iter_batched_ref(
-                || {
-                    T::new(
-                        CryptoConfig::new(CipherSuite::default(), ec_group, sig_type),
-                        handshake_type,
-                        Default::default(),
-                    )
-                },
-                |harness| {
-                    // harnesses with certain parameters fail to initialize for
-                    // some past versions of s2n-tls, but missing data can be
-                    // visually interpolated in the historical performance graph
-                    if let Ok(harness) = harness {
-                        let _ = harness.handshake();
+    for handshake_type in HandshakeType::iter() {
+        for kx_group in KXGroup::iter() {
+            for sig_type in SigType::iter() {
+                let mut bench_group = c.benchmark_group(match handshake_type {
+                    HandshakeType::ServerAuth => format!("handshake-{:?}-{:?}", kx_group, sig_type),
+                    HandshakeType::MutualAuth => {
+                        format!("handshake-mTLS-{:?}-{:?}", kx_group, sig_type)
                     }
-                },
-                BatchSize::SmallInput,
-            )
-        });
-    }
-
-    for handshake_type in [ServerAuth, MutualAuth] {
-        for ec_group in [SECP256R1, X25519] {
-            for sig_type in [Rsa2048, Rsa3072, Rsa4096, Ec384] {
-                let mut bench_group = c.benchmark_group(format!(
-                    "handshake-{:?}-{:?}-{:?}",
-                    handshake_type, ec_group, sig_type
-                ));
-                bench_handshake_for_library::<S2NHarness>(
+                });
+                bench_handshake_for_library::<S2NConnection>(
                     &mut bench_group,
-                    "s2n-tls",
                     handshake_type,
-                    ec_group,
+                    kx_group,
                     sig_type,
                 );
                 #[cfg(not(feature = "historical-perf"))]
                 {
-                    bench_handshake_for_library::<RustlsHarness>(
+                    bench_handshake_for_library::<RustlsConnection>(
                         &mut bench_group,
-                        "rustls",
                         handshake_type,
-                        ec_group,
+                        kx_group,
                         sig_type,
                     );
-                    bench_handshake_for_library::<OpenSslHarness>(
+                    bench_handshake_for_library::<OpenSslConnection>(
                         &mut bench_group,
-                        "openssl",
                         handshake_type,
-                        ec_group,
+                        kx_group,
                         sig_type,
                     );
                 }

bindings/rust/bench/benches/throughput.rs

@@ -2,68 +2,64 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use bench::{
-    CipherSuite::{self, *},
-    CryptoConfig, ECGroup, HandshakeType, OpenSslHarness, RustlsHarness, S2NHarness, SigType,
-    TlsBenchHarness, harness::ConnectedBuffer,
+    harness::ConnectedBuffer, CipherSuite, CryptoConfig, HandshakeType, KXGroup, OpenSslConnection,
+    RustlsConnection, S2NConnection, SigType, TlsConnPair, TlsConnection,
 };
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion,
     Throughput,
 };
+use strum::IntoEnumIterator;
+
+fn bench_throughput_for_library<T: TlsConnection>(
+    bench_group: &mut BenchmarkGroup<WallTime>,
+    shared_buf: &mut [u8],
+    cipher_suite: CipherSuite,
+) {
+    bench_group.bench_function(T::name(), |b| {
+        b.iter_batched_ref(
+            || {
+                TlsConnPair::<T, T>::new(
+                    CryptoConfig::new(cipher_suite, KXGroup::default(), SigType::default()),
+                    HandshakeType::default(),
+                    ConnectedBuffer::default(),
+                )
+                .map(|mut h| {
+                    let _ = h.handshake();
+                    h
+                })
+            },
+            |conn_pair_res| {
+                if let Ok(conn_pair) = conn_pair_res {
+                    let _ = conn_pair.round_trip_transfer(shared_buf);
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    });
+}
 
 pub fn bench_throughput_cipher_suite(c: &mut Criterion) {
     // arbitrarily large to cut across TLS record boundaries
     let mut shared_buf = [0u8; 100000];
 
-    fn bench_throughput_for_library<T: TlsBenchHarness>(
-        bench_group: &mut BenchmarkGroup<WallTime>,
-        name: &str,
-        shared_buf: &mut [u8],
-        cipher_suite: CipherSuite,
-    ) {
-        bench_group.bench_function(name, |b| {
-            b.iter_batched_ref(
-                || {
-                    T::new(
-                        CryptoConfig::new(cipher_suite, ECGroup::default(), SigType::default()),
-                        HandshakeType::default(),
-                        ConnectedBuffer::default(),
-                    )
-                    .map(|mut h| {
-                        let _ = h.handshake();
-                        h
-                    })
-                },
-                |harness| {
-                    if let Ok(harness) = harness {
-                        let _ = harness.round_trip_transfer(shared_buf);
-                    }
-                },
-                BatchSize::SmallInput,
-            )
-        });
-    }
-
-    for cipher_suite in [AES_128_GCM_SHA256, AES_256_GCM_SHA384] {
+    for cipher_suite in CipherSuite::iter() {
         let mut bench_group = c.benchmark_group(format!("throughput-{:?}", cipher_suite));
         bench_group.throughput(Throughput::Bytes(shared_buf.len() as u64));
-        bench_throughput_for_library::<S2NHarness>(
+        bench_throughput_for_library::<S2NConnection>(
             &mut bench_group,
-            "s2n-tls",
             &mut shared_buf,
             cipher_suite,
         );
         #[cfg(not(feature = "historical-perf"))]
         {
-            bench_throughput_for_library::<RustlsHarness>(
+            bench_throughput_for_library::<RustlsConnection>(
                 &mut bench_group,
-                "rustls",
                 &mut shared_buf,
                 cipher_suite,
             );
-            bench_throughput_for_library::<OpenSslHarness>(
+            bench_throughput_for_library::<OpenSslConnection>(
                 &mut bench_group,
-                "openssl",
                 &mut shared_buf,
                 cipher_suite,
             );

bindings/rust/bench/certs/generate_certs.sh

@@ -16,11 +16,13 @@ pushd "$(dirname "$0")" > /dev/null
 # Generates certs with given algorithms and bits in $1$2/, ex. ec384/
 # $1: rsa or ec
 # $2: number of bits
+# $3: directory under the `certs/` directory to put certs in
 cert-gen () {
     echo -e "\n----- generating certs for $1$2 -----\n"
 
     key_family=$1
     key_size=$2
+    dir_name=$3
 
     # set openssl argument name
     if [[ $key_family == rsa ]]; then
@@ -30,8 +32,8 @@ cert-gen () {
     fi
 
     # make directory for certs
-    mkdir -p $key_family$key_size
-    cd $key_family$key_size
+    mkdir -p $dir_name
+    cd $dir_name
 
     echo "generating CA private key and certificate"
     openssl req -new -nodes -x509 -newkey $key_family -pkeyopt $argname$key_size -keyout  ca-key.pem -out ca-cert.pem -days 65536 -config ../config/ca.cnf
@@ -62,13 +64,13 @@ cert-gen () {
 
 if [[ $1 != "clean" ]] 
 then
-    cert-gen ec 384
-    cert-gen rsa 2048
-    cert-gen rsa 3072
-    cert-gen rsa 4096
+    cert-gen ec 384 ecdsa384
+    cert-gen rsa 2048 rsa2048
+    cert-gen rsa 3072 rsa3072
+    cert-gen rsa 4096 rsa4096
 else 
     echo "cleaning certs"
-    rm -rf ec*/ rsa*/
+    rm -rf ecdsa*/ rsa*/
 fi
 
 popd > /dev/null

bindings/rust/bench/src/bin/graph_memory.rs

@@ -28,7 +28,7 @@ fn get_bytes_from_snapshot(name: &str, i: i32) -> i32 {
 }
 
 /// Get the difference in bytes between two snapshots, which is memory of the
-/// `i`th TlsBenchHarness (client and server)
+/// `i`th TlsConnPair (client and server)
 fn get_bytes_diff(name: &str, i: i32) -> i32 {
     get_bytes_from_snapshot(name, i + 1) - get_bytes_from_snapshot(name, i)
 }

bindings/rust/bench/src/bin/memory.rs

@@ -1,18 +1,21 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-use bench::{harness::ConnectedBuffer, OpenSslHarness, RustlsHarness, S2NHarness, TlsBenchHarness};
+use bench::{
+    harness::ConnectedBuffer, CryptoConfig, HandshakeType, OpenSslConnection, RustlsConnection,
+    S2NConnection, TlsConnPair, TlsConnection,
+};
 use std::{fs::create_dir_all, path::Path};
 
-fn memory_bench<T: TlsBenchHarness>(dir_name: &str) {
+fn memory_bench<T: TlsConnection>(dir_name: &str) {
     println!("testing {dir_name}");
 
     if !Path::new(&format!("target/memory/{dir_name}")).is_dir() {
         create_dir_all(format!("target/memory/{dir_name}")).unwrap();
     }
 
-    let mut harnesses = Vec::new();
-    harnesses.reserve(100);
+    let mut conn_pairs = Vec::new();
+    conn_pairs.reserve(100);
 
     // reserve space for buffers before benching
     let mut buffers = Vec::new();
@@ -22,27 +25,27 @@ fn memory_bench<T: TlsBenchHarness>(dir_name: &str) {
     }
 
     // handshake one harness to initalize libraries
-    let mut harness = T::default().unwrap();
-    harness.handshake().unwrap();
+    let mut conn_pair = TlsConnPair::<T, T>::default();
+    conn_pair.handshake().unwrap();
 
     // tell massif to take initial memory snapshot
     crabgrind::monitor_command(format!("snapshot target/memory/{dir_name}/0.snapshot")).unwrap();
 
-    // make and handshake 100 harness
+    // make and handshake 100 connection pairs
     // memory usage stabilizes after first few handshakes
     for i in 1..101 {
         // put new harness directly into harness vec
-        harnesses.push(
-            T::new(
-                Default::default(),
-                Default::default(),
+        conn_pairs.push(
+            TlsConnPair::<T, T>::new(
+                CryptoConfig::default(),
+                HandshakeType::default(),
                 buffers.pop().unwrap(), // take ownership of buffer
             )
             .unwrap(),
         );
 
         // handshake last harness added
-        harnesses
+        conn_pairs
             .as_mut_slice()
             .last_mut()
             .unwrap()
@@ -58,7 +61,7 @@ fn memory_bench<T: TlsBenchHarness>(dir_name: &str) {
 fn main() {
     assert!(!cfg!(debug_assertions), "need to run in release mode");
 
-    memory_bench::<S2NHarness>("s2n-tls");
-    memory_bench::<RustlsHarness>("rustls");
-    memory_bench::<OpenSslHarness>("openssl");
+    memory_bench::<S2NConnection>("s2n-tls");
+    memory_bench::<RustlsConnection>("rustls");
+    memory_bench::<OpenSslConnection>("openssl");
 }