Squashed commit of the following:

commit eafb8a2 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 18:05:50 2023 -0400 shell -> bash commit 12071b8 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 17:59:09 2023 -0400 add ubuntu quickstart back to readme commit 10bf557 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 17:52:06 2023 -0400 fixes commit 74adf8d Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 16:46:19 2023 -0400 fixes commit 0548d07 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 16:43:08 2023 -0400 consolidate commit cbe8f2d Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 14:55:30 2023 -0400 remove old doc sections commit f194321 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 12:25:28 2023 -0400 more content commit 882eb1d Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 09:08:24 2023 -0400 fixes commit ce37d0e Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Mon Jul 31 09:03:45 2023 -0400 fixes commit 011d15f Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Sat Jul 29 22:59:51 2023 -0400 cmake consuming commit 7feadc1 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Sat Jul 29 22:27:02 2023 -0400 fixes commit 2914950 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Sat Jul 29 21:34:24 2023 -0400 traditional make commit 02f9841 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Sat Jul 29 19:45:43 2023 -0400 s2n-tls build section commit 86c4983 Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Sat Jul 29 11:56:32 2023 -0400 Update build documentation commit ea6d02a Author: Sam Clark <3758302+goatgoose@users.noreply.github.com> Date: Fri Jul 28 16:49:21 2023 -0400 bindings: release 0.0.35 (aws#4122) commit 35d08ba Author: Justin Zhang <76919968+tinzh@users.noreply.github.com> Date: Fri Jul 28 12:31:21 2023 -0700 refactor(bench): separate out client and server connections in benching harness (aws#4113) Enables more better control of connections for benching experiments commit 65e74ca Author: Lindsay Stewart <slindsay@amazon.com> Date: Wed Jul 26 02:26:40 2023 -0700 Print error for 32bit test (aws#4107) commit b0b253e Author: toidiu <apoorv@toidiu.com> Date: Wed Jul 26 00:30:44 2023 -0700 ktls: set keys on socket and enable ktls (aws#4071) commit 403d5e6 Author: Lindsay Stewart <slindsay@amazon.com> Date: Tue Jul 25 16:03:09 2023 -0700 Trying to use an invalid ticket should not mutate state (aws#4110) commit bce2b1a Author: James Mayclin <maycj@amazon.com> Date: Tue Jul 25 14:44:33 2023 -0700 fix: get_session behavior for TLS 1.3 (aws#4104) commit 6881358 Author: Justin Zhang <76919968+tinzh@users.noreply.github.com> Date: Tue Jul 25 10:10:21 2023 -0700 feat(bench): add different certificate signature algorithms to benchmarks (aws#4080) commit aab13d5 Author: Justin Zhang <76919968+tinzh@users.noreply.github.com> Date: Mon Jul 24 18:17:30 2023 -0700 feat(bench): add memory bench with valgrind/massif (aws#4081) commit 20b0174 Author: Justin Zhang <76919968+tinzh@users.noreply.github.com> Date: Mon Jul 24 13:26:32 2023 -0700 feat(bench): add historical performance benchmark (aws#4083) commit 5cc827d Author: Doug Chapman <54039637+dougch@users.noreply.github.com> Date: Thu Jul 20 11:50:50 2023 -0700 nix: pin corretto version (aws#4103)
goatgoose · Jul 31, 2023 · a5b1db2 · a5b1db2
1 parent 57943c8
commit a5b1db2
Show file tree

Hide file tree

Showing 53 changed files with 3,545 additions and 1,220 deletions.
diff --git a/README.md b/README.md
@@ -14,54 +14,29 @@ s2n-tls is a C99 implementation of the TLS/SSL protocols that is designed to be
 [![Join the chat at https://gitter.im/awslabs/s2n](https://badges.gitter.im/awslabs/s2n.svg)](https://gitter.im/awslabs/s2n?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 ## Quickstart for Ubuntu
-1. Fork s2n-tls on GitHub
-2. Run the following commands on Ubuntu.
-```
-git clone https://github.com/${YOUR_GITHUB_ACCOUNT_NAME}/s2n-tls.git
-cd s2n-tls
-
-# Pick an "env" line from the codebuild/codebuild.config file and run it, in this case choose the openssl-1.1.1 with GCC 9 build
-S2N_LIBCRYPTO=openssl-1.1.1 BUILD_S2N=true TESTS=integrationv2 GCC_VERSION=9
-
-sudo codebuild/bin/s2n_install_test_dependencies.sh
-codebuild/bin/s2n_codebuild.sh
-```
-
-## Quickstart for OSX (or other platforms)
-
-If you are building on OSX, or simply don't want to execute the entire build script above, you can use build tools like Ninja.
 
-### OSX
-
-An example of building on OSX:
+```bash
+# clone s2n-tls
+git clone https://github.com/aws/s2n-tls.git
+cd s2n-tls
 
-```sh
-# Install required dependencies using homebrew
-brew install ninja cmake coreutils openssl@1.1
+# install build dependencies
+sudo apt update
+sudo apt install cmake ninja-build
 
-# Clone the s2n-tls source repository into the `s2n-tls` directory
-git clone https://github.com/${YOUR_GITHUB_ACCOUNT_NAME}/s2n-tls.git
-cd s2n-tls
+# install a libcrypto
+sudo apt install libssl-dev
 
-# Create a build directory, and build s2n-tls with debug symbols and a specific OpenSSL version.
+# build s2n-tls
 cmake . -Bbuild -GNinja \
-    -DCMAKE_BUILD_TYPE=Debug \
-    -DCMAKE_PREFIX_PATH=$(dirname $(dirname $(brew list openssl@1.1|grep libcrypto.dylib)))
-cmake --build ./build -j $(nproc)
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=./s2n-tls-install
+ninja -C build -j $(nproc)
 CTEST_PARALLEL_LEVEL=$(nproc) ninja -C build test
+ninja -C build install
 ```
 
-### Amazonlinux2
-
-Install dependencies with `./codebuild/bin/install_al2_dependencies.sh` after cloning.
-
-```sh
-git clone https://github.com/${YOUR_GITHUB_ACCOUNT_NAME}/s2n-tls.git
-cd s2n-tls
-cmake . -Bbuild -DCMAKE_EXE_LINKER_FLAGS="-lcrypto -lz" -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
-cmake --build ./build -j $(nproc)
-CTEST_PARALLEL_LEVEL=$(nproc) make -C build test
-```
+See the [s2n-tls build documentation](docs/BUILD.md) for further guidance on building s2n-tls for your platform.
 
 ## Have a Question?
 If you have any questions about Submitting PR's, Opening Issues, s2n-tls API usage, or something similar, we have a public chatroom available here to answer your questions: https://gitter.im/awslabs/s2n

diff --git a/api/s2n.h b/api/s2n.h
@@ -2214,6 +2214,9 @@ S2N_API extern int s2n_session_ticket_get_lifetime(struct s2n_session_ticket *ti
 /**
  * De-serializes the session state and updates the connection accordingly.
  *
+ * If this method fails, the connection should not be affected: calling s2n_negotiate
+ * with the connection should simply result in a full handshake.
+ *
  * @param conn A pointer to the s2n_connection object
  * @param session A pointer to a buffer of size `length`
  * @param length The size of the `session` buffer

diff --git a/bindings/rust/bench/.gitignore b/bindings/rust/bench/.gitignore
@@ -0,0 +1,3 @@
+*.pem
+*.svg
+!historical-perf-*.svg
diff --git a/bindings/rust/bench/Cargo.toml b/bindings/rust/bench/Cargo.toml
@@ -3,16 +3,26 @@ name = "bench"
 version = "0.1.0"
 edition = "2021"
 
+[features]
+historical-perf = []
+
 [dependencies]
 s2n-tls = { path = "../s2n-tls" }
 rustls = "0.21"
 rustls-pemfile = "1.0"
-openssl = "0.10"
+openssl = { version = "0.10", features = ["vendored"] }
 errno = "0.3"
 libc = "0.2"
+crabgrind = "0.1"
+rand = "0.8"
+rand_distr = "0.4"
+plotters = "0.3"
+serde_json = "1.0"
+semver = "1.0"
+strum = { version = "0.25", features = ["derive"] }
 
 [dev-dependencies]
-criterion = "0.3"
+criterion = "0.5"
 
 [[bench]]
 name = "handshake"

diff --git a/bindings/rust/bench/README.md b/bindings/rust/bench/README.md
@@ -4,12 +4,37 @@ We use to Criterion.rs to benchmark s2n-tls against two commonly used TLS librar
 
 ## Setup 
 
-Setup is easy! Just have OpenSSL installed and generate Rust bindings for s2n-tls using `bindings/rust/generate.sh`.
+Setup is easy! Just have OpenSSL installed, generate Rust bindings for s2n-tls using `../generate.sh`, and generate certs using `certs/generate_certs.sh`. 
+
+Dependencies are the same as with s2n-tls. Currently, this crate has only been tested on Ubuntu (both x86 and ARM), but we expect everything to work with other Unix environments. 
+
+To bench with AWS-LC, Amazon's custom libcrypto implementation, first run `install-aws-lc.sh` to install AWS-LC for the bench crate. To then run the benchmarks with AWS-LC, use Cargo with either the flag `--config aws-lc-config/s2n.toml` or `--config aws-lc-config/rustls.toml` (or both). You can also append these configs to `.cargo/config.toml` to let Cargo automatically detect the settings without specifying the flags each time.  
+
+For example, to get started with benching s2n-tls with AWS-LC:
+
+```
+../generate.sh
+certs/generate_certs.sh
+./install-aws-lc.sh
+cargo bench --config aws-lc-config/s2n.toml
+```
 
 ## Running benchmarks
 
 The benchmarks can be run with the `cargo bench` command. Criterion will auto-generate an HTML report in `target/criterion/`. 
 
+To run memory benchmarks, run `memory/bench-memory.sh`. A graph of memory usage will be generated in `memory/memory.svg`.
+
+## Historical benchmarks
+
+To do historical benchmarks, run `historical-perf/bench-past.sh`. This will checkout old versions of s2n-tls back to v1.3.16 in `target/` and run benchmarks on those with the `historical-perf` feature, disabling Rustls and OpenSSL benches.
+
+### Caveats
+
+The last version benched is v1.3.16, since before that, the s2n-tls Rust bindings have a different API and would thus require a different bench harness to test. 
+
+v1.3.30-1.3.37 are not benched because of depedency issues when generating the Rust bindings. However, versions before and after are benched, so the overall trend in performance can still be seen without the data from these versions.
+
 ## Implementation details
 
 We use Rust bindings for s2n-tls and OpenSSL. All of our benchmarks are run in Rust on a single thread for consistency. 
@@ -20,8 +45,25 @@ To remove external factors, we use custom IO with our benchmarks, bypassing the
 
 ### Certificate generation
 
-All certs are stored in `certs/` and can be regenerated using `certs/generate_certs.sh`. There is one root cert that directly signs the server and client certs that are used in benchmarking. Currently, we use ECDSA with `secp384r1`.
+There is one root cert that directly signs the server and client certs that are used in benchmarking. We currently bench RSA and ECDSA certs.
 
 ### Negotiation parameters
 
 The cipher suites benchmarked are `TLS_AES_128_GCM_SHA256` and `TLS_AES_256_GCM_SHA384`, and the key exchange methods benchmarked are ECDHE with `secp256r1` and with `x25519`. We also test connections with and without client authentication (mTLS).
+
+## Sample output
+
+### Historical performance
+
+Because these benches take a longer time to generate (>30 min), we include the results from historical benching (as of v1.3.47) here.
+
+Notes: 
+- Two sets of parameters for the handshake couldn't be benched before 1.3.40, since security policies that negotiated those policies as their top choice did not exist before then.
+- There is no data from 1.3.30 to 1.3.37 because those versions have a dependency issue that cause the Rust bindings not to build. However, there is data before and after that period, so the performance for those versions can be inferred via interpolation.
+- The improvement in throughput in 1.3.28 was most likely caused by the addition of LTO to the default Rust bindings build. 
+- Since the benches are run over a long time, noise on the machine can cause variability, as seen in the throughput graph.
+- The variability can be seen with throughput especially because it is calculated as the inverse of time taken.
+
+![historical-perf-handshake](images/historical-perf-handshake.svg)
+
+![historical-perf-throughput](images/historical-perf-throughput.svg)
diff --git a/bindings/rust/bench/aws-lc-config/rustls.toml b/bindings/rust/bench/aws-lc-config/rustls.toml
@@ -0,0 +1,2 @@
+[patch.crates-io]
+rustls = { path = "target/rustls/rustls" }
diff --git a/bindings/rust/bench/aws-lc-config/s2n.toml b/bindings/rust/bench/aws-lc-config/s2n.toml
@@ -0,0 +1,3 @@
+[env]
+S2N_TLS_LIB_DIR = "target/s2n-tls-build/lib"
+LD_LIBRARY_PATH = "target/s2n-tls-build/lib"
diff --git a/bindings/rust/bench/benches/handshake.rs b/bindings/rust/bench/benches/handshake.rs
@@ -2,57 +2,76 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use bench::{
-    CryptoConfig,
-    ECGroup::{self, *},
-    HandshakeType::{self, *},
-    OpenSslHarness, RustlsHarness, S2NHarness, TlsBenchHarness,
+    CipherSuite, CryptoConfig, HandshakeType, KXGroup, OpenSslConnection, RustlsConnection,
+    S2NConnection, SigType, TlsConnPair, TlsConnection,
 };
 use criterion::{
     criterion_group, criterion_main, measurement::WallTime, BatchSize, BenchmarkGroup, Criterion,
 };
-use std::any::type_name;
+use strum::IntoEnumIterator;
+
+fn bench_handshake_for_library<T: TlsConnection>(
+    bench_group: &mut BenchmarkGroup<WallTime>,
+    handshake_type: HandshakeType,
+    kx_group: KXGroup,
+    sig_type: SigType,
+) {
+    // generate all harnesses (TlsConnPair structs) beforehand so that benchmarks
+    // only include negotiation and not config/connection initialization
+    bench_group.bench_function(T::name(), |b| {
+        b.iter_batched_ref(
+            || {
+                TlsConnPair::<T, T>::new(
+                    CryptoConfig::new(CipherSuite::default(), kx_group, sig_type),
+                    handshake_type,
+                    Default::default(),
+                )
+            },
+            |conn_pair_res| {
+                // harnesses with certain parameters fail to initialize for
+                // some past versions of s2n-tls, but missing data can be
+                // visually interpolated in the historical performance graph
+                if let Ok(conn_pair) = conn_pair_res {
+                    let _ = conn_pair.handshake();
+                }
+            },
+            BatchSize::SmallInput,
+        )
+    });
+}
 
 pub fn bench_handshake_params(c: &mut Criterion) {
-    fn bench_handshake_for_library<T: TlsBenchHarness>(
-        bench_group: &mut BenchmarkGroup<WallTime>,
-        handshake_type: HandshakeType,
-        ec_group: ECGroup,
-    ) {
-        bench_group.bench_function(type_name::<T>(), |b| {
-            b.iter_batched_ref(
-                || {
-                    T::new(
-                        CryptoConfig {
-                            cipher_suite: Default::default(),
-                            ec_group,
-                        },
+    for handshake_type in HandshakeType::iter() {
+        for kx_group in KXGroup::iter() {
+            for sig_type in SigType::iter() {
+                let mut bench_group = c.benchmark_group(match handshake_type {
+                    HandshakeType::ServerAuth => format!("handshake-{:?}-{:?}", kx_group, sig_type),
+                    HandshakeType::MutualAuth => {
+                        format!("handshake-mTLS-{:?}-{:?}", kx_group, sig_type)
+                    }
+                });
+                bench_handshake_for_library::<S2NConnection>(
+                    &mut bench_group,
+                    handshake_type,
+                    kx_group,
+                    sig_type,
+                );
+                #[cfg(not(feature = "historical-perf"))]
+                {
+                    bench_handshake_for_library::<RustlsConnection>(
+                        &mut bench_group,
                         handshake_type,
-                    )
-                    .unwrap()
-                },
-                |harness| {
-                    harness.handshake().unwrap();
-                },
-                BatchSize::SmallInput,
-            )
-        });
-    }
-
-    for handshake_type in [ServerAuth, MutualAuth] {
-        for ec_group in [SECP256R1, X25519] {
-            let mut bench_group =
-                c.benchmark_group(format!("handshake-{:?}-{:?}", handshake_type, ec_group));
-            bench_handshake_for_library::<S2NHarness>(&mut bench_group, handshake_type, ec_group);
-            bench_handshake_for_library::<RustlsHarness>(
-                &mut bench_group,
-                handshake_type,
-                ec_group,
-            );
-            bench_handshake_for_library::<OpenSslHarness>(
-                &mut bench_group,
-                handshake_type,
-                ec_group,
-            );
+                        kx_group,
+                        sig_type,
+                    );
+                    bench_handshake_for_library::<OpenSslConnection>(
+                        &mut bench_group,
+                        handshake_type,
+                        kx_group,
+                        sig_type,
+                    );
+                }
+            }
         }
     }
 }