Move each implementation's benchmarks into a separate crate.

.travis.yml

@@ -36,4 +36,4 @@ script:
   # break the build.
   - $CC --version
   - $CXX --version
-  - cargo test --release --target=$TARGET --verbose
+  - ./cargo_all test --release --target=$TARGET --verbose

README.md

@@ -4,41 +4,46 @@
 
 ```
 git clone https://github.com/briansmith/crypto-bench
-cargo bench
+./cargo_all bench
 ```
 
 You must use Rust Nightly because `cargo bench` is used for these benchmarks,
-and only Right Nightly supports `cargo bench`. Note that currently the
-benchmarks do not build for pc-windows-msvc targets because
-[rust-fastpbkdf2 fails to build for pc-windows-msvc](https://github.com/ctz/rust-fastpbkdf2/issues/1)
-and [rust-crypto fails to build for pc-windows-msvc](https://github.com/DaGenix/rust-crypto/issues/324).
-pc-windows-gnu targets are not supported because the pc-windows-gnu targets are
-a dead end.
+and only Right Nightly supports `cargo bench`.
 
 You don't need to run `cargo build`, and in fact `cargo build` does not do
 anything useful for this crate.
 
-`cargo test` runs one iteration of every benchmark. This is useful for quickly
-making sure that a change to the benchmarks does not break them.
-
-Note that some crypto libraries only support a subset of the tests.
+`./cargo_all.sh test` runs one iteration of every benchmark for every
+implementation. This is useful for quickly making sure that a change to the
+benchmarks does not break them. Do this before submitting a pull request.
 
 ## How to run all the benchmarks for a specific crypto library
 
-Use `cargo bench ::<implementation>::`:
-
-* `cargo bench ::ring::` runs all the tests for [*ring*](https://github.com/briansmith/ring).
-* `cargo bench ::octavo::` runs all the tests for [Octavo](https://github.com/libOctavo/octavo).
-* `cargo bench ::rust_crypto::` runs all the tests for [rust-crypto](https://github.com/DaGenix/rust-crypto).
-* `cargo bench ::fastpbkdf2::` runs all the tests for [rust-fastpbkdf2](https://github.com/ctz/rust-fastpbkdf2).
-* `cargo bench ::openssl::` runs all the tests for [rust-openssl](https://github.com/sfackler/rust-openssl).
+* `(cd fastpbkdf2 && cargo bench)` runs all the tests for [rust-fastpbkdf2](https://github.com/ctz/rust-fastpbkdf2).
+* `(cd octavo && cargo bench)` runs all the tests for [Octavo](https://github.com/libOctavo/octavo).
+* `(cd openssl && cargo bench)` runs all the tests for [rust-openssl](https://github.com/sfackler/rust-openssl).
+* `(cd ring && cargo bench)` runs all the tests for [*ring*](https://github.com/briansmith/ring).
+* `(cd rust_crypto && cargo bench)` runs all the tests for [rust-crypto](https://github.com/DaGenix/rust-crypto).
 
 ## How to run other subsets of the benchmarks
 
-`cargo bench` takes arbitrary substrings of the test names as parameters, so
+`./cargo bench` takes arbitrary substrings of the test names as parameters, so
 you can get as specific as you want. For example,
-`digest::ring::sha512::_2000` will run just the SHA-512
-benchmark that takes a 2000 byte input, just for *ring*.
+`./cargo_all bench sha512::_2000` will run just the SHA-512
+benchmark that takes a 2000 byte input, for every implementation.
+
+## Why does each implementation's benchmark live in a separate crate?
+
+* Not all implementations build and work on all platforms. And, some
+  implementations requre manual configuration (e.g. building/installing some
+  third-party C library) to work. The `cargo_all` scripts keep going on
+  failure, so they'll build/test/benchmark whatever implementations actually
+  work, and skip over the ones that don't. This would be difficult to acheive
+  if all the benchmarks were in one crate.
+
+* Some implementations (*ring* and any of the crates that use OpenSSL) cannot
+  (correctly) coexist in the same program because they define extern C symbols
+  with the same names, but which have different ABIs.
 
 ## How to contribute
 

cargo_all

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+(cd fastpbkdf2 && cargo $*)
+(cd octavo && cargo $*)
+(cd openssl && cargo $*)
+(cd ring && cargo $*)
+(cd rust_crypto && cargo $*)

cargo_all.bat

@@ -0,0 +1,19 @@
+PUSHD fastpbkdf2
+cargo %*
+POPD
+
+PUSHD octavo
+cargo %*
+POPD
+
+PUSHD openssl
+cargo %*
+POPD
+
+PUSHD ring
+cargo %*
+POPD
+
+PUSHD rust_crypto
+cargo %*
+POPD

crypto_bench/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+authors = ["Brian Smith <brian@briansmith.org>"]
+description = "Benchmarks of Rust crypto libraries."
+name = "crypto_bench"
+version = "0.1.0"
+
+[lib]
+name = "crypto_bench"
+path = "crypto_bench.rs"
+
+# Ensure that the bench, release, and test settings are the same.
+
+[profile.bench]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[profile.release]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[profile.test]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1

crypto_bench/crypto_bench.rs

@@ -0,0 +1,86 @@
+// This crate only contains benchmarks, but we have to define a program with
+// `main` to avoid things complaining.
+
+pub mod aead {
+    // All the AEADs we're testing use 96-bit nonces.
+    pub const NONCE: [u8; 96 / 8] = [0u8; 96 / 8];
+
+    // A TLS 1.2 finished message is always 12 bytes long.
+    pub const TLS12_FINISHED_LEN: usize = 12;
+
+    // A TLS 1.3 finished message is "[t]he size of the HMAC output for the
+    // Hash used for the handshake," which is usually SHA-256.
+    pub const TLS13_FINISHED_LEN: usize = 32;
+
+    // In TLS 1.2, 13 bytes of additional data are used for AEAD cipher suites.
+    pub const TLS12_AD: [u8; 13] = [
+        23,         // Type: application_data
+        3, 3,       // Version = TLS 1.2.
+        0x12, 0x34, // Length = 0x1234.
+        0, 0, 0, 0, 0, 0, 0, 1, // Record #1
+    ];
+
+    // In TLS 1.3, no additional data is used for AEAD cipher suites.
+    pub const TLS13_AD: [u8; 0] = [ ];
+}
+
+pub const SHA1_BLOCK_LEN: usize = 512 / 8;
+pub const SHA1_OUTPUT_LEN: usize = 160 / 8;
+pub const SHA256_BLOCK_LEN: usize = 512 / 8;
+pub const SHA256_OUTPUT_LEN: usize = 256 / 8;
+pub const SHA384_BLOCK_LEN: usize = 1024 / 8;
+pub const SHA384_OUTPUT_LEN: usize = 384 / 8;
+pub const SHA512_BLOCK_LEN: usize = 1024 / 8;
+pub const SHA512_OUTPUT_LEN: usize = 512 / 8;
+
+#[macro_export]
+macro_rules! digest_bench {
+    ( $bench_fn_name:ident, $input_len:expr, $input:ident,
+      $calculation:expr) => {
+        #[bench]
+        fn $bench_fn_name(b: &mut test::Bencher) {
+            let $input = vec![0u8; $input_len];
+            let $input = &$input[..];
+            b.bytes = $input_len as u64;
+            b.iter(|| $calculation);
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! digest_benches {
+    ($block_len:expr, $input:ident, $calculation:expr) =>
+    {
+        use test;
+
+        digest_bench!(block_len, $block_len, $input, $calculation); // PBKDF2
+        digest_bench!(_16, 16, $input, $calculation); // BoringSSL
+        digest_bench!(_256, 256, $input, $calculation); // BoringSSL
+        digest_bench!(_1000, 1000, $input, $calculation); // X.509 TBSCertificate
+        digest_bench!(_2000, 2000, $input, $calculation); // X.509 TBSCertificate
+        digest_bench!(_8192, 8192, $input, $calculation); // BoringSSL
+    }
+}
+
+pub mod pbkdf2 {
+    // These values are copied from
+    // https://github.com/ctz/rust-fastpbkdf2/tree/master/pbkdf2-bench, except
+    // `ITERATIONS` was lowered from `1 << 20` because the benchmarks were
+    // excruciatingly slow with 2^20 iterations, and that iteration count isn't
+    // realistic for most applications anyway.
+    pub const ITERATIONS: u32 = 100_000;
+    pub const PASSWORD: &'static [u8] = b"password";
+    pub const PASSWORD_STR: &'static str = "password";
+    pub const SALT: &'static [u8] = b"salt";
+}
+
+#[macro_export]
+macro_rules! pbkdf2_bench {
+    ( $bench_fn_name:ident, $out_len:expr, $out:ident, $calculation:expr) => {
+        #[bench]
+        fn $bench_fn_name(b: &mut test::Bencher) {
+            let mut $out = [0u8; $out_len];
+            b.iter(|| $calculation)
+        }
+    }
+}

fastpbkdf2/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+authors = ["Brian Smith <brian@briansmith.org>"]
+name = "crypto_bench_fastpbkdf2"
+version = "0.1.0"
+
+[lib]
+name = "crypto_bench_fastpbkdf2"
+path = "fastpbkdf2.rs"
+
+[dependencies.crypto_bench]
+path = "../crypto_bench"
+
+[dependencies]
+fastpbkdf2 = "0.1"
+
+# Ensure that the bench, release, and test settings are the same.
+
+[profile.bench]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[profile.release]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[profile.test]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1

fastpbkdf2/fastpbkdf2.rs

@@ -0,0 +1,33 @@
+#![feature(test)]
+
+extern crate test;
+
+#[macro_use]
+extern crate crypto_bench;
+
+extern crate fastpbkdf2;
+
+#[cfg(test)]
+mod pbkdf2 {
+    use crypto_bench;
+    use fastpbkdf2;
+    use test;
+
+    pbkdf2_bench!(hmac_sha1, crypto_bench::SHA1_OUTPUT_LEN, out,
+                  fastpbkdf2::pbkdf2_hmac_sha1(crypto_bench::pbkdf2::PASSWORD,
+                                               crypto_bench::pbkdf2::SALT,
+                                               crypto_bench::pbkdf2::ITERATIONS,
+                                               &mut out));
+
+    pbkdf2_bench!(hmac_sha256, crypto_bench::SHA256_OUTPUT_LEN, out,
+                  fastpbkdf2::pbkdf2_hmac_sha256(crypto_bench::pbkdf2::PASSWORD,
+                                                 crypto_bench::pbkdf2::SALT,
+                                                 crypto_bench::pbkdf2::ITERATIONS,
+                                                 &mut out));
+
+    pbkdf2_bench!(hmac_sha512, crypto_bench::SHA512_OUTPUT_LEN, out,
+                  fastpbkdf2::pbkdf2_hmac_sha512(crypto_bench::pbkdf2::PASSWORD,
+                                                 crypto_bench::pbkdf2::SALT,
+                                                 crypto_bench::pbkdf2::ITERATIONS,
+                                                 &mut out));
+}

octavo/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+authors = ["Brian Smith <brian@briansmith.org>"]
+name = "crypto_bench_octavo"
+version = "0.1.0"
+
+[lib]
+name = "crypto_bench_octavo"
+path = "octavo.rs"
+
+[dependencies.crypto_bench]
+path = "../crypto_bench"
+
+[dependencies.octavo]
+git = "https://github.com/libOctavo/octavo"
+
+# Ensure that the bench, release, and test settings are the same.
+
+[profile.bench]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[profile.release]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[profile.test]
+opt-level = 3
+debug = true
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1

octavo/octavo.rs

@@ -0,0 +1,41 @@
+#![feature(test)]
+
+extern crate test;
+
+#[macro_use]
+extern crate crypto_bench;
+
+extern crate octavo;
+
+#[cfg(test)]
+mod digest {
+    macro_rules! octavo_digest_benches {
+        ( $name:ident, $block_len:expr, $output_len:expr, $digest:expr) => {
+            mod $name {
+                use crypto_bench;
+                use octavo::digest;
+                use octavo::digest::Digest;
+
+                digest_benches!($block_len, input, {
+                    let mut out = [0u8; $output_len];
+                    let mut ctx = $digest;
+                    ctx.update(&input);
+                    ctx.result(&mut out[..]);
+                });
+            }
+        }
+    }
+
+    octavo_digest_benches!(sha1, crypto_bench::SHA1_BLOCK_LEN,
+                           crypto_bench::SHA1_OUTPUT_LEN,
+                           digest::sha1::Sha1::default());
+    octavo_digest_benches!(sha256, crypto_bench::SHA256_BLOCK_LEN,
+                           crypto_bench::SHA256_OUTPUT_LEN,
+                           digest::sha2::Sha256::default());
+    octavo_digest_benches!(sha384, crypto_bench::SHA384_BLOCK_LEN,
+                           crypto_bench::SHA384_OUTPUT_LEN,
+                           digest::sha2::Sha384::default());
+    octavo_digest_benches!(sha512, crypto_bench::SHA512_BLOCK_LEN,
+                           crypto_bench::SHA512_OUTPUT_LEN,
+                           digest::sha2::Sha512::default());
+}