Add optional async/await support. #403

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Added support for benchmarking async functions
+
 ### Fixed
 - Criterion.rs will now give a clear error message in case of benchmarks that take zero time.
 - Added some extra code to ensure that every sample has at least one iteration.

Cargo.toml

@@ -31,6 +31,10 @@ num-traits = { version = "0.2", default-features = false }
 oorandom = "11.1"
 rayon = "1.3"
 regex = { version = "1.3", default-features = false, features = ["std"] }
+futures = { version = "0.3", default_features = false, optional = true }
+smol = { version = "1.2", default-features = false, optional = true }
+tokio = { version = "1.0", default-features = false, features = ["rt"], optional = true }
+async-std = { version = "1.9", optional = true }
 
 [dependencies.plotters]
 version = "^0.2.12"
@@ -42,16 +46,29 @@ tempfile = "3.1"
 approx = "0.3"
 quickcheck = { version = "0.9", default-features = false }
 rand = "0.7"
+futures = { version = "0.3", default_features = false, features = ["executor"] }
 
 [badges]
 travis-ci = { repository = "bheisler/criterion.rs" }
 appveyor = { repository = "bheisler/criterion.rs", id = "4255ads9ctpupcl2" }
 maintenance = { status = "passively-maintained" }
 
 [features]
-real_blackbox = []
 default = []
 
+# Enable use of the nightly-only test::black_box function to discourage compiler optimizations.
+real_blackbox = []
+
+# Enable async/await support
+async = ["futures"]
+
+# These features enable built-in support for running async benchmarks on each different async 
+# runtime.
+async_futures = ["futures/executor", "async"]
+async_smol = ["smol", "async"]
+async_tokio = ["tokio", "async"]
+async_std = ["async-std", "async"]
+
 [workspace]
 exclude = ["cargo-criterion"]
 
@@ -61,3 +78,7 @@ harness = false
 
 [lib]
 bench = false
+
+# Enable all of the async runtimes for the docs.rs output
+[package.metadata.docs.rs]
+features = ["async_futures", "async_smol", "async_std", "async_tokio"]

benches/bench_main.rs

@@ -13,4 +13,5 @@ criterion_main! {
     benchmarks::measurement_overhead::benches,
     benchmarks::custom_measurement::benches,
     benchmarks::sampling_mode::benches,
+    benchmarks::async_measurement_overhead::benches,
 }

benches/benchmarks/async_measurement_overhead.rs

@@ -0,0 +1,48 @@
+use criterion::{async_executor::FuturesExecutor, criterion_group, BatchSize, Criterion};
+
+fn some_benchmark(c: &mut Criterion) {
+    let mut group = c.benchmark_group("async overhead");
+    group.bench_function("iter", |b| b.to_async(FuturesExecutor).iter(|| async { 1 }));
+    group.bench_function("iter_with_setup", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_with_setup(|| (), |_| async { 1 })
+    });
+    group.bench_function("iter_with_large_setup", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_with_large_setup(|| (), |_| async { 1 })
+    });
+    group.bench_function("iter_with_large_drop", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_with_large_drop(|| async { 1 })
+    });
+    group.bench_function("iter_batched_small_input", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_batched(|| (), |_| async { 1 }, BatchSize::SmallInput)
+    });
+    group.bench_function("iter_batched_large_input", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_batched(|| (), |_| async { 1 }, BatchSize::LargeInput)
+    });
+    group.bench_function("iter_batched_per_iteration", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_batched(|| (), |_| async { 1 }, BatchSize::PerIteration)
+    });
+    group.bench_function("iter_batched_ref_small_input", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_batched_ref(|| (), |_| async { 1 }, BatchSize::SmallInput)
+    });
+    group.bench_function("iter_batched_ref_large_input", |b| {
+        b.to_async(FuturesExecutor)
+            .iter_batched_ref(|| (), |_| async { 1 }, BatchSize::LargeInput)
+    });
+    group.bench_function("iter_batched_ref_per_iteration", |b| {
+        b.to_async(FuturesExecutor).iter_batched_ref(
+            || (),
+            |_| async { 1 },
+            BatchSize::PerIteration,
+        )
+    });
+    group.finish();
+}
+
+criterion_group!(benches, some_benchmark);

benches/benchmarks/mod.rs

@@ -8,3 +8,14 @@ pub mod measurement_overhead;
 pub mod sampling_mode;
 pub mod special_characters;
 pub mod with_inputs;
+
+#[cfg(feature = "async_futures")]
+pub mod async_measurement_overhead;
+
+#[cfg(not(feature = "async_futures"))]
+pub mod async_measurement_overhead {
+    use criterion::{criterion_group, BatchSize, Criterion};
+    fn some_benchmark(c: &mut Criterion) {}
+
+    criterion_group!(benches, some_benchmark);
+}

book/src/SUMMARY.md

@@ -18,6 +18,7 @@
     - [Custom Measurements](./user_guide/custom_measurements.md)
     - [Profiling](./user_guide/profiling.md)
     - [Custom Test Framework](./user_guide/custom_test_framework.md)
+    - [Benchmarking async functions](./user_guide/benchmarking_async.md)
 - [cargo-criterion](./cargo_criterion/cargo_criterion.md)
   - [Configuring cargo-criterion](./cargo_criterion/configuring_cargo_criterion.md)
   - [External Tools](./cargo_criterion/external_tools.md)

book/src/user_guide/benchmarking_async.md

@@ -0,0 +1,59 @@
+## Benchmarking async functions
+
+As of version 0.3.4, Criterion.rs has optional support for benchmarking async functions.
+Benchmarking async functions works just like benchmarking regular functions, except that the
+caller must provide a futures executor to run the benchmark in.
+
+### Example:
+
+```rust
+use criterion::BenchmarkId;
+use criterion::Criterion;
+use criterion::{criterion_group, criterion_main};
+
+// This is a struct that tells Criterion.rs to use the "futures" crate's current-thread executor
+use criterion::async_executor::FuturesExecutor;
+
+// Here we have an async function to benchmark
+async fn do_something(size: usize) {
+    // Do something async with the size
+}
+
+fn from_elem(c: &mut Criterion) {
+    let size: usize = 1024;
+
+    c.bench_with_input(BenchmarkId::new("input_example", size), &size, |b, &s| {
+        // Insert a call to `to_async` to convert the bencher to async mode.
+        // The timing loops are the same as with the normal bencher.
+        b.to_async(FuturesExecutor).iter(|| do_something(s));
+    });
+}
+
+criterion_group!(benches, from_elem);
+criterion_main!(benches);
+```
+
+As can be seen in the code above, to benchmark async functions we must provide an async runtime to
+the bencher to run the benchmark in. The runtime structs are listed in the table below.
+
+### Enabling Async Benchmarking
+
+To enable async benchmark support, Criterion.rs must be compiled with one or more of the following
+features, depending on which futures executor(s) you want to benchmark on. It is recommended to use
+the same executor that you would use in production. If your executor is not listed here, you can
+implement the `criterion::async_executor::AsyncExecutor` trait for it to add support, or send a pull
+request.
+
+| Crate     | Feature                       | Executor Struct                                       |
+| --------- | ----------------------------- | ----------------------------------------------------- |
+| Tokio     | "async_tokio"                 | `tokio::runtime::Runtime`, `&tokio::runtime::Runtime` |
+| async-std | "async_std" (note underscore) | `AsyncStdExecutor`                                    |
+| Smol      | "async_smol"                  | `SmolExecutor`                                        |
+| futures   | "async_futures"               | `FuturesExecutor`                                     |
+| Other     | "async"                       |                                                       |
+
+### Considerations when benchmarking async functions
+
+Async functions naturally result in more measurement overhead than synchronous functions. It is
+recommended to prefer synchronous functions when benchmarking where possible, especially for small
+functions.

ci/script.sh

@@ -2,11 +2,13 @@ set -ex
 
 export CARGO_INCREMENTAL=0
 
+FEATURES="async_smol async_tokio async_std async_futures"
+
 if [ "$CLIPPY" = "yes" ]; then
       cargo clippy --all -- -D warnings
 elif [ "$DOCS" = "yes" ]; then
     cargo clean
-    cargo doc --all --no-deps
+    cargo doc --features "$FEATURES" --all --no-deps
     cd book
     mdbook build
     cd ..
@@ -20,10 +22,10 @@ elif [ "$MINIMAL_VERSIONS" = "yes" ]; then
 else
     export RUSTFLAGS="-D warnings"
 
-    cargo build $BUILD_ARGS
+    cargo build --features "$FEATURES" $BUILD_ARGS
 
-    cargo test --all
-    cargo test --benches
+    cargo test --features "$FEATURES" --all
+    cargo test --features "$FEATURES" --benches
 
     cd bencher_compat
     export CARGO_TARGET_DIR="../target"

src/async_executor.rs

@@ -0,0 +1,66 @@
+//! This module defines a trait that can be used to plug in different Futures executors into
+//! Criterion.rs' async benchmarking support.
+//!
+//! Implementations are provided for:
+//! * Tokio (implemented directly for tokio::Runtime)
+//! * Async-std
+//! * Smol
+//! * The Futures crate
+//!
+//! Please note that async benchmarks will have a small amount of measurement overhead relative
+//! to synchronous benchmarks. It is recommended to use synchronous benchmarks where possible, to
+//! improve measurement accuracy.
+
+use std::future::Future;
+
+/// Plugin trait used to allow benchmarking on multiple different async runtimes.
+///
+/// Smol, Tokio and Async-std are supported out of the box, as is the current-thread runner from the
+/// Futures crate; it is recommended to use whichever runtime you use in production.
+pub trait AsyncExecutor {
+    /// Spawn the given future onto this runtime and block until it's complete, returning the result.
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T;
+}
+
+/// Runs futures on the 'futures' crate's built-in current-thread executor
+#[cfg(feature = "async_futures")]
+pub struct FuturesExecutor;
+#[cfg(feature = "async_futures")]
+impl AsyncExecutor for FuturesExecutor {
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        futures::executor::block_on(future)
+    }
+}
+
+/// Runs futures on the 'soml' crate's global executor
+#[cfg(feature = "async_smol")]
+pub struct SmolExecutor;
+#[cfg(feature = "async_smol")]
+impl AsyncExecutor for SmolExecutor {
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        smol::block_on(future)
+    }
+}
+
+#[cfg(feature = "async_tokio")]
+impl AsyncExecutor for tokio::runtime::Runtime {
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        self.block_on(future)
+    }
+}
+#[cfg(feature = "async_tokio")]
+impl AsyncExecutor for &tokio::runtime::Runtime {
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        (*self).block_on(future)
+    }
+}
+
+/// Runs futures on the 'async-std' crate's global executor
+#[cfg(feature = "async_std")]
+pub struct AsyncStdExecutor;
+#[cfg(feature = "async_std")]
+impl AsyncExecutor for AsyncStdExecutor {
+    fn block_on<T>(&self, future: impl Future<Output = T>) -> T {
+        async_std::task::block_on(future)
+    }
+}