Version 0.4.0

.github/workflows/ci.yaml

@@ -5,6 +5,7 @@ on:
   pull_request:
     branches:
       - master
+      - version-0.4
 
 name: tests
 
@@ -23,32 +24,64 @@ jobs:
       - uses: actions/checkout@v2
 
       - uses: actions-rs/toolchain@v1
+        name: Setup rust toolchain
         with:
           profile: minimal
           toolchain: ${{ matrix.rust }}
           override: true
           components: rustfmt, clippy
 
       - uses: Swatinem/rust-cache@v1
+        name: Load dependencies from cache
 
       - uses: actions-rs/cargo@v1
+        name: Build with stable features
         with:
           command: build
           args: --features stable
 
       - uses: actions-rs/cargo@v1
+        if: ${{ matrix.rust == 'nightly' }}
+        name: Build with unstable features
+        with:
+          command: build
+          args: --all-features
+
+      - uses: actions-rs/cargo@v1
+        name: Build with minimal features
+        with:
+          command: build
+          args: --no-default-features
+
+      - uses: actions-rs/cargo@v1
+        name: Test with stable features
         with:
           command: test
           args: --features stable
 
       - uses: actions-rs/cargo@v1
+        name: Test with minimal features
+        with:
+          command: test
+          args: --no-default-features
+
+      - uses: actions-rs/cargo@v1
+        name: Check for non-standard formatting
         if: ${{ matrix.rust == 'stable' }}
         with:
           command: fmt
           args: --all -- --check
 
       - uses: actions-rs/cargo@v1
-        if: ${{ matrix.rust != '1.40.0' }} # 1.40 has horrible lints.
+        name: Check for clippy hints
         with:
           command: clippy
           args: -- -D warnings
+
+      - name: Test run targeting WASI
+        run: |
+          curl https://wasmtime.dev/install.sh -sSf | bash
+          source ~/.bashrc
+          export PATH=$HOME/.wasmtime/bin/:$PATH
+          cargo install cargo-wasi
+          cargo wasi bench --no-default-features -- --test

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [0.4.0] - IN DEVELOPMENT
+
+### Removed
+- The `Criterion::can_plot` function has been removed.
+- The `Criterion::bench_function_over_inputs` function has been removed.
+- The `Criterion::bench_functions` function has been removed.
+- The `Criterion::bench` function has been removed.
+
+### Changed
+- HTML report hidden behind non-default feature flag: 'html_reports'
+- Standalone support (ie without cargo-criterion) feature flag: 'cargo_bench_support'
+
 ## [Unreleased]
 
 ## [0.3.5] - 2021-07-26

Cargo.toml

@@ -17,6 +17,7 @@ license     = "Apache-2.0/MIT"
 exclude     = ["book/*"]
 
 [dependencies]
+anes           = "0.1"
 lazy_static    = "1.4"
 criterion-plot = { path = "plot", version = "0.4.4" }
 itertools      = "0.10"
@@ -26,21 +27,24 @@ serde_derive   = "1.0"
 serde_cbor     = "0.11"
 atty           = "0.2"
 clap           = { version = "2.33", default-features = false }
-csv            = "1.1"
 walkdir        = "2.3"
 tinytemplate   = "1.1"
 cast           = "0.2"
-num-traits     = { version = "0.2", default-features = false }
+num-traits     = { version = "0.2", default-features = false, features = ["std"] }
 oorandom       = "11.1"
-rayon          = "1.3"
 regex          = { version = "1.3", default-features = false, features = ["std"] }
-futures        = { version = "0.3", default_features = false, optional = true }
-smol           = { version = "1.2", default-features = false, optional = true }
-tokio          = { version = "1.0", default-features = false, features = ["rt"], optional = true }
-async-std      = { version = "1.9", optional = true }
+
+# Optional dependencies
+rayon     = { version = "1.3", optional = true }
+csv       = { version = "1.1", optional = true }
+futures   = { version = "0.3", default_features = false, optional = true }
+smol      = { version = "1.2", default-features = false, optional = true }
+tokio     = { version = "1.0", default-features = false, features = ["rt"], optional = true }
+async-std = { version = "1.9", optional = true }
 
 [dependencies.plotters]
 version          = "^0.3.1"
+optional         = true
 default-features = false
 features         = ["svg_backend", "area_series", "line_series"]
 
@@ -55,8 +59,15 @@ futures    = { version = "0.3", default_features = false, features = ["executor"
 maintenance = { status = "passively-maintained" }
 
 [features]
-stable  = ["async_futures", "async_smol", "async_tokio", "async_std"]
-default = ["cargo_bench_support"]
+stable  = [
+  "csv_output",
+  "html_reports",
+  "async_futures",
+  "async_smol",
+  "async_tokio",
+  "async_std",
+]
+default = ["rayon", "plotters", "cargo_bench_support"]
 
 # Enable use of the nightly-only test::black_box function to discourage compiler optimizations.
 real_blackbox = []
@@ -82,7 +93,7 @@ cargo_bench_support = []
 # This feature _currently_ does nothing, but in 0.4.0 it will be
 # required in order to have Criterion.rs generate CSV files. This feature is deprecated in favor of
 # cargo-criterion's --message-format=json option.
-csv_output = []
+csv_output = ["csv"]
 
 [workspace]
 exclude = ["cargo-criterion"]

benches/benchmarks/compare_functions.rs

@@ -1,6 +1,4 @@
-#![allow(deprecated)]
-
-use criterion::{criterion_group, BenchmarkId, Criterion, Fun, ParameterizedBenchmark};
+use criterion::{criterion_group, BenchmarkId, Criterion};
 
 fn fibonacci_slow(n: u64) -> u64 {
     match n {
@@ -27,23 +25,10 @@ fn fibonacci_fast(n: u64) -> u64 {
 }
 
 fn compare_fibonaccis(c: &mut Criterion) {
-    let fib_slow = Fun::new("Recursive", |b, i| b.iter(|| fibonacci_slow(*i)));
-    let fib_fast = Fun::new("Iterative", |b, i| b.iter(|| fibonacci_fast(*i)));
-
-    let functions = vec![fib_slow, fib_fast];
+    let mut group = c.benchmark_group("Fibonacci");
 
-    c.bench_functions("Fibonacci", functions, 20);
-}
-fn compare_fibonaccis_builder(c: &mut Criterion) {
-    c.bench(
-        "Fibonacci2",
-        ParameterizedBenchmark::new(
-            "Recursive",
-            |b, i| b.iter(|| fibonacci_slow(*i)),
-            vec![20u64, 21u64],
-        )
-        .with_function("Iterative", |b, i| b.iter(|| fibonacci_fast(*i))),
-    );
+    group.bench_with_input("Recursive", &20, |b, i| b.iter(|| fibonacci_slow(*i)));
+    group.bench_with_input("Iterative", &20, |b, i| b.iter(|| fibonacci_fast(*i)));
 }
 fn compare_fibonaccis_group(c: &mut Criterion) {
     let mut group = c.benchmark_group("Fibonacci3");
@@ -58,28 +43,4 @@ fn compare_fibonaccis_group(c: &mut Criterion) {
     group.finish()
 }
 
-fn compare_looped(c: &mut Criterion) {
-    use criterion::black_box;
-
-    c.bench(
-        "small",
-        ParameterizedBenchmark::new("unlooped", |b, i| b.iter(|| i + 10), vec![10]).with_function(
-            "looped",
-            |b, i| {
-                b.iter(|| {
-                    for _ in 0..10_000 {
-                        black_box(i + 10);
-                    }
-                })
-            },
-        ),
-    );
-}
-
-criterion_group!(
-    fibonaccis,
-    compare_fibonaccis,
-    compare_fibonaccis_builder,
-    compare_fibonaccis_group,
-    compare_looped
-);
+criterion_group!(fibonaccis, compare_fibonaccis, compare_fibonaccis_group,);

benches/benchmarks/external_process.rs

@@ -14,7 +14,6 @@ fn create_command() -> Command {
     command
 }
 
-#[allow(deprecated)]
 fn python_fibonacci(c: &mut Criterion) {
     let has_python3 = Command::new("python3")
         .arg("--version")

benches/benchmarks/iter_with_large_drop.rs

@@ -1,28 +1,22 @@
-#![allow(deprecated)]
-
-use criterion::{criterion_group, Benchmark, Criterion, Throughput};
+use criterion::{criterion_group, Criterion, Throughput};
 use std::time::Duration;
 
 const SIZE: usize = 1024 * 1024;
 
 fn large_drop(c: &mut Criterion) {
-    c.bench(
-        "iter_with_large_drop",
-        Benchmark::new("large_drop", |b| {
-            let v: Vec<_> = (0..SIZE).map(|i| i as u8).collect();
-            b.iter_with_large_drop(|| v.clone());
-        })
-        .throughput(Throughput::Bytes(SIZE as u64)),
-    );
+    let mut group = c.benchmark_group("iter_with_large_drop");
+    group.throughput(Throughput::Bytes(SIZE as u64));
+    group.bench_function("large_drop", |b| {
+        let v: Vec<_> = (0..SIZE).map(|i| i as u8).collect();
+        b.iter_with_large_drop(|| v.clone());
+    });
 }
 
 fn small_drop(c: &mut Criterion) {
-    c.bench(
-        "iter_with_large_drop",
-        Benchmark::new("small_drop", |b| {
-            b.iter_with_large_drop(|| SIZE);
-        }),
-    );
+    let mut group = c.benchmark_group("iter_with_large_drop");
+    group.bench_function("small_drop", |b| {
+        b.iter_with_large_drop(|| SIZE);
+    });
 }
 
 fn short_warmup() -> Criterion {

benches/benchmarks/iter_with_large_setup.rs

@@ -1,32 +1,25 @@
-#![allow(deprecated)]
-
-use criterion::{criterion_group, Benchmark, Criterion, Throughput};
+use criterion::{criterion_group, BatchSize, Criterion, Throughput};
 use std::time::Duration;
 
 const SIZE: usize = 1024 * 1024;
 
 fn large_setup(c: &mut Criterion) {
-    c.bench(
-        "iter_with_large_setup",
-        Benchmark::new("large_setup", |b| {
-            // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead.
-            b.iter_with_large_setup(
-                || (0..SIZE).map(|i| i as u8).collect::<Vec<_>>(),
-                |v| v.clone(),
-            )
-        })
-        .throughput(Throughput::Bytes(SIZE as u64)),
-    );
+    let mut group = c.benchmark_group("iter_with_large_setup");
+    group.throughput(Throughput::Bytes(SIZE as u64));
+    group.bench_function("large_setup", |b| {
+        b.iter_batched(
+            || (0..SIZE).map(|i| i as u8).collect::<Vec<_>>(),
+            |v| v.clone(),
+            BatchSize::NumBatches(1),
+        )
+    });
 }
 
 fn small_setup(c: &mut Criterion) {
-    c.bench(
-        "iter_with_large_setup",
-        Benchmark::new("small_setup", |b| {
-            // NOTE: iter_with_large_setup is deprecated. Use iter_batched instead.
-            b.iter_with_large_setup(|| SIZE, |size| size)
-        }),
-    );
+    let mut group = c.benchmark_group("iter_with_large_setup");
+    group.bench_function("small_setup", |b| {
+        b.iter_batched(|| SIZE, |size| size, BatchSize::NumBatches(1))
+    });
 }
 
 fn short_warmup() -> Criterion {

benches/benchmarks/measurement_overhead.rs

@@ -5,7 +5,7 @@ fn some_benchmark(c: &mut Criterion) {
     group.bench_function("iter", |b| b.iter(|| 1));
     group.bench_function("iter_with_setup", |b| b.iter_with_setup(|| (), |_| 1));
     group.bench_function("iter_with_large_setup", |b| {
-        b.iter_with_large_setup(|| (), |_| 1)
+        b.iter_batched(|| (), |_| 1, BatchSize::NumBatches(1))
     });
     group.bench_function("iter_with_large_drop", |b| b.iter_with_large_drop(|| 1));
     group.bench_function("iter_batched_small_input", |b| {

plot/src/data.rs

@@ -1,5 +1,3 @@
-#![allow(deprecated)]
-
 use std::mem;
 
 use cast::From as _0;

src/analysis/mod.rs

@@ -26,7 +26,7 @@ macro_rules! elapsed {
         info!(
             "{} took {}",
             $msg,
-            crate::format::time(crate::DurationExt::to_nanos(elapsed) as f64)
+            crate::format::time(elapsed.as_nanos() as f64)
         );
 
         out
@@ -128,7 +128,7 @@ pub(crate) fn common<M: Measurement, T: ?Sized>(
         .collect::<Vec<f64>>();
     let avg_times = Sample::new(&avg_times);
 
-    if criterion.connection.is_none() && criterion.load_baseline.is_none() {
+    if criterion.should_save_baseline() {
         log_if_err!({
             let mut new_dir = criterion.output_directory.clone();
             new_dir.push(id.as_directory_name());
@@ -139,7 +139,7 @@ pub(crate) fn common<M: Measurement, T: ?Sized>(
 
     let data = Data::new(&iters, &times);
     let labeled_sample = tukey::classify(avg_times);
-    if criterion.connection.is_none() {
+    if criterion.should_save_baseline() {
         log_if_err!({
             let mut tukey_file = criterion.output_directory.to_owned();
             tukey_file.push(id.as_directory_name());
@@ -156,7 +156,7 @@ pub(crate) fn common<M: Measurement, T: ?Sized>(
         distributions.slope = Some(distribution);
     }
 
-    if criterion.connection.is_none() && criterion.load_baseline.is_none() {
+    if criterion.should_save_baseline() {
         log_if_err!({
             let mut sample_file = criterion.output_directory.clone();
             sample_file.push(id.as_directory_name());
@@ -237,7 +237,7 @@ pub(crate) fn common<M: Measurement, T: ?Sized>(
         criterion.measurement.formatter(),
     );
 
-    if criterion.connection.is_none() && criterion.load_baseline.is_none() {
+    if criterion.should_save_baseline() {
         log_if_err!({
             let mut benchmark_file = criterion.output_directory.clone();
             benchmark_file.push(id.as_directory_name());
@@ -365,5 +365,6 @@ fn copy_new_dir_to_base(id: &str, baseline: &str, output_directory: &Path) {
         &new_dir.join("benchmark.json"),
         &base_dir.join("benchmark.json")
     ));
+    #[cfg(feature = "csv_output")]
     try_else_return!(fs::cp(&new_dir.join("raw.csv"), &base_dir.join("raw.csv")));
 }