Merge #1698

1698: Fix benchmarks r=pksunkara a=CreepySkeleton



Co-authored-by: CreepySkeleton <creepy-skeleton@yandex.ru>

.travis.yml

@@ -26,8 +26,7 @@ branches:
 jobs:
   allow_failures:
     - rust: nightly
-    - env:
-        - SHARD=coverage
+    - name: Coverage
   fast_finish: true
   include:
     - os: osx
@@ -36,8 +35,7 @@ jobs:
     - {}
     - rust: beta
     - rust: nightly
-    - env:
-        - SHARD=lint
+    - name: Linting (fmt + clippy)
       before_script:
         - rustup component add clippy
         - rustup component add rustfmt
@@ -46,13 +44,10 @@ jobs:
         - cargo clippy --lib --features "yaml unstable" -- -D warnings
         - cargo clippy --tests --examples --features "yaml unstable"
         - cargo fmt -- --check
-    - rust: nightly
-      env:
-        - SHARD=bench
+    - name: Benchmarking
       script:
         - cargo bench
-    - env:
-        - SHARD=coverage
+    - name: Coverage
       addons:
         apt:
           packages:

Cargo.toml

@@ -26,6 +26,30 @@ categories = ["command-line-interface"]
 license = "MIT OR Apache-2.0"
 readme = "README.md"
 
+[[bench]]
+harness = false
+name = "01_default"
+
+[[bench]]
+harness = false
+name = "02_simple"
+
+[[bench]]
+harness = false
+name = "03_complex"
+
+[[bench]]
+harness = false
+name = "04_new_help"
+
+[[bench]]
+harness = false
+name = "05_ripgrep"
+
+[[bench]]
+harness = false
+name = "06_rustup"
+
 [badges]
 travis-ci = { repository = "clap-rs/clap", branch = "master" }
 azure-devops = { project = "clap-rs/clap", pipeline = "clap-rs.clap" }
@@ -54,6 +78,7 @@ ansi_term = { version = "0.12.1", optional = true }
 regex = "1.0"
 lazy_static = "1"
 version-sync = "0.8"
+criterion = "0.3"
 
 [features]
 default     = ["suggestions", "color", "vec_map", "derive", "std"]

benches/01_default.rs

@@ -1,14 +1,15 @@
-#![feature(test)]
-
 use clap::App;
-use test::Bencher;
+use criterion::{criterion_group, criterion_main, Criterion};
 
-#[bench]
-fn build_app(b: &mut Bencher) {
-    b.iter(|| App::new("claptests"));
+pub fn empty_app(c: &mut Criterion) {
+    c.bench_function("build_app", |b| b.iter(|| App::new("claptests")));
 }
 
-#[bench]
-fn parse_clean(b: &mut Bencher) {
-    b.iter(|| App::new("claptests").get_matches_from(vec![""]));
+pub fn parse_clean(c: &mut Criterion) {
+    c.bench_function("parse_clean", |b| {
+        b.iter(|| App::new("claptests").get_matches_from(vec![""]))
+    });
 }
+
+criterion_group!(benches, empty_app, parse_clean);
+criterion_main!(benches);

benches/02_simple.rs

@@ -1,7 +1,5 @@
-#![feature(test)]
-
 use clap::{App, Arg};
-use test::Bencher;
+use criterion::{criterion_group, criterion_main, Criterion};
 
 macro_rules! create_app {
     () => {{
@@ -15,95 +13,92 @@ macro_rules! create_app {
     }};
 }
 
-#[bench]
-fn build_app(b: &mut Bencher) {
-    b.iter(|| create_app!());
+pub fn build_app(c: &mut Criterion) {
+    c.bench_function("build_app", |b| b.iter(|| create_app!()));
 }
 
-#[bench]
-fn add_flag(b: &mut Bencher) {
-    fn build_app() -> App<'static> {
-        App::new("claptests")
-    }
-
-    b.iter(|| build_app().arg(Arg::from("-s, --some 'something'")));
+pub fn add_flag(c: &mut Criterion) {
+    c.bench_function("add_flag", |b| {
+        b.iter(|| App::new("claptests").arg(Arg::from("-s, --some 'something'")))
+    });
 }
 
-#[bench]
-fn add_flag_ref(b: &mut Bencher) {
-    fn build_app() -> App<'static> {
-        App::new("claptests")
-    }
-
-    b.iter(|| {
-        let arg = Arg::from("-s, --some 'something'");
-        build_app().arg(&arg)
+pub fn add_flag_ref(c: &mut Criterion) {
+    c.bench_function("add_flag_ref", |b| {
+        b.iter(|| {
+            let arg = Arg::from("-s, --some 'something'");
+            App::new("claptests").arg(&arg)
+        })
     });
 }
 
-#[bench]
-fn add_opt(b: &mut Bencher) {
-    fn build_app() -> App<'static> {
-        App::new("claptests")
-    }
-
-    b.iter(|| build_app().arg(Arg::from("-s, --some <FILE> 'something'")));
+pub fn add_opt(c: &mut Criterion) {
+    c.bench_function("add_opt", |b| {
+        b.iter(|| App::new("claptests").arg(Arg::from("-s, --some <FILE> 'something'")))
+    });
 }
 
-#[bench]
-fn add_opt_ref(b: &mut Bencher) {
-    fn build_app() -> App<'static> {
-        App::new("claptests")
-    }
-
-    b.iter(|| {
-        let arg = Arg::from("-s, --some <FILE> 'something'");
-        build_app().arg(&arg)
+pub fn add_opt_ref(c: &mut Criterion) {
+    c.bench_function("add_opt_ref", |b| {
+        b.iter(|| {
+            let arg = Arg::from("-s, --some <FILE> 'something'");
+            App::new("claptests").arg(&arg)
+        })
     });
 }
 
-#[bench]
-fn add_pos(b: &mut Bencher) {
-    fn build_app() -> App<'static> {
-        App::new("claptests")
-    }
-
-    b.iter(|| build_app().arg(Arg::with_name("some")));
+pub fn add_pos(c: &mut Criterion) {
+    c.bench_function("add_pos", |b| {
+        b.iter(|| App::new("claptests").arg(Arg::with_name("some")))
+    });
 }
 
-#[bench]
-fn add_pos_ref(b: &mut Bencher) {
-    fn build_app() -> App<'static> {
-        App::new("claptests")
-    }
-
-    b.iter(|| {
-        let arg = Arg::with_name("some");
-        build_app().arg(&arg)
+pub fn add_pos_ref(c: &mut Criterion) {
+    c.bench_function("add_pos_ref", |b| {
+        b.iter(|| {
+            let arg = Arg::with_name("some");
+            App::new("claptests").arg(&arg)
+        })
     });
 }
 
-#[bench]
-fn parse_clean(b: &mut Bencher) {
-    b.iter(|| create_app!().get_matches_from(vec![""]));
+pub fn parse_flag(c: &mut Criterion) {
+    c.bench_function("parse_flag", |b| {
+        b.iter(|| create_app!().get_matches_from(vec!["myprog", "-f"]))
+    });
 }
 
-#[bench]
-fn parse_flag(b: &mut Bencher) {
-    b.iter(|| create_app!().get_matches_from(vec!["myprog", "-f"]));
+pub fn parse_option(c: &mut Criterion) {
+    c.bench_function("parse_option", |b| {
+        b.iter(|| create_app!().get_matches_from(vec!["myprog", "-o", "option1"]))
+    });
 }
 
-#[bench]
-fn parse_option(b: &mut Bencher) {
-    b.iter(|| create_app!().get_matches_from(vec!["myprog", "-o", "option1"]));
+pub fn parse_positional(c: &mut Criterion) {
+    c.bench_function("parse_positional", |b| {
+        b.iter(|| create_app!().get_matches_from(vec!["myprog", "arg1"]))
+    });
 }
 
-#[bench]
-fn parse_positional(b: &mut Bencher) {
-    b.iter(|| create_app!().get_matches_from(vec!["myprog", "arg1"]));
+pub fn parse_complex(c: &mut Criterion) {
+    c.bench_function("parse_complex", |b| {
+        b.iter(|| create_app!().get_matches_from(vec!["myprog", "-o", "option1", "-f", "arg1"]))
+    });
 }
 
-#[bench]
-fn parse_complex(b: &mut Bencher) {
-    b.iter(|| create_app!().get_matches_from(vec!["myprog", "-o", "option1", "-f", "arg1"]));
-}
+criterion_group!(
+    benches,
+    parse_complex,
+    parse_positional,
+    parse_option,
+    parse_flag,
+    add_pos_ref,
+    add_pos,
+    add_opt_ref,
+    add_opt,
+    add_flag_ref,
+    add_flag,
+    build_app
+);
+
+criterion_main!(benches);