Added options to specify the max/exact numbers of runs.

Fixes sharkdp#77.

src/hyperfine/app.rs

@@ -54,6 +54,22 @@ fn build_app() -> App<'static, 'static> {
                 .value_name("NUM")
                 .help("Perform at least NUM runs for each command (default: 10)."),
         )
+        .arg(
+            Arg::with_name("max-runs")
+                .long("max-runs")
+                .short("M")
+                .takes_value(true)
+                .value_name("NUM")
+                .help("Perform at most NUM runs for each command."),
+        )
+        .arg(
+            Arg::with_name("runs")
+                .long("runs")
+                .short("r")
+                .takes_value(true)
+                .value_name("NUM")
+                .help("Perform exactly NUM runs for each command."),
+        )
         .arg(
             Arg::with_name("prepare")
                 .long("prepare")

src/hyperfine/benchmark.rs

@@ -1,3 +1,4 @@
+use std::cmp;
 use std::io;
 use std::process::Stdio;
 
@@ -187,7 +188,7 @@ pub fn run_benchmark(
 
     // Set up progress bar (and spinner for initial measurement)
     let progress_bar = get_progress_bar(
-        options.min_runs,
+        options.runs.min,
         "Initial time measurement",
         &options.output_style,
     );
@@ -215,11 +216,10 @@ pub fn run_benchmark(
         / (res.time_real + prepare_res.time_real + shell_spawning_time.time_real))
         as u64;
 
-    let count = if runs_in_min_time >= options.min_runs {
-        runs_in_min_time
-    } else {
-        options.min_runs
-    };
+    let count = cmp::min(
+        cmp::max(runs_in_min_time, options.runs.min),
+        options.runs.max
+    );
 
     let count_remaining = count - 1;
 

src/hyperfine/error.rs

@@ -9,8 +9,20 @@ pub enum ParameterScanError {
     TooLarge,
 }
 
-impl ParameterScanError {
-    fn __description(&self) -> &str {
+impl From<num::ParseIntError> for ParameterScanError {
+    fn from(e: num::ParseIntError) -> ParameterScanError {
+        ParameterScanError::ParseIntError(e)
+    }
+}
+
+impl fmt::Display for ParameterScanError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.description())
+    }
+}
+
+impl Error for ParameterScanError {
+    fn description(&self) -> &str {
         match *self {
             ParameterScanError::ParseIntError(ref e) => e.description(),
             ParameterScanError::EmptyRange => "Empty parameter range",
@@ -19,20 +31,23 @@ impl ParameterScanError {
     }
 }
 
-impl From<num::ParseIntError> for ParameterScanError {
-    fn from(e: num::ParseIntError) -> ParameterScanError {
-        ParameterScanError::ParseIntError(e)
-    }
+#[derive(Debug)]
+pub enum OptionsError {
+    RunsBelowTwo,
+    EmptyRunsRange,
 }
 
-impl fmt::Display for ParameterScanError {
+impl fmt::Display for OptionsError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.__description())
+        write!(f, "{}", self.description())
     }
 }
 
-impl Error for ParameterScanError {
+impl Error for OptionsError {
     fn description(&self) -> &str {
-        self.__description()
+        match *self {
+            OptionsError::EmptyRunsRange => "Empty runs range",
+            OptionsError::RunsBelowTwo => "Number of runs below two",
+        }
     }
 }

src/hyperfine/types.rs

@@ -73,13 +73,31 @@ pub enum OutputStyleOption {
     NoColor,
 }
 
+/// Number of runs for a benchmark
+pub struct Runs {
+    /// Minimum number of benchmark runs
+    pub min: u64,
+
+    /// Maximum number of benchmark runs
+    pub max: u64,
+}
+
+impl Default for Runs {
+    fn default() -> Runs {
+        Runs {
+            min: 10,
+            max: ::std::u64::MAX,
+        }
+    }
+}
+
 /// A set of options for hyperfine
 pub struct HyperfineOptions {
     /// Number of warmup runs
     pub warmup_count: u64,
 
-    /// Minimum number of benchmark runs
-    pub min_runs: u64,
+    /// Number of benchmark runs
+    pub runs: Runs,
 
     /// Minimum benchmarking time
     pub min_time_sec: Second,
@@ -101,7 +119,7 @@ impl Default for HyperfineOptions {
     fn default() -> HyperfineOptions {
         HyperfineOptions {
             warmup_count: 0,
-            min_runs: 10,
+            runs: Runs::default(),
             min_time_sec: 3.0,
             failure_action: CmdFailureAction::RaiseError,
             preparation_command: None,

src/main.rs

@@ -42,7 +42,7 @@ mod hyperfine;
 
 use hyperfine::app::get_arg_matches;
 use hyperfine::benchmark::{mean_shell_spawning_time, run_benchmark};
-use hyperfine::error::ParameterScanError;
+use hyperfine::error::{ParameterScanError, OptionsError};
 use hyperfine::export::{ExportManager, ExportType};
 use hyperfine::internal::write_benchmark_comparison;
 use hyperfine::types::{
@@ -95,7 +95,10 @@ fn main() {
     let export_manager = build_export_manager(&matches);
     let commands = build_commands(&matches);
 
-    let res = run(&commands, &options);
+    let res = match options {
+        Ok(opts) => run(&commands, &opts),
+        Err(e) => error(e.description()),
+    };
 
     match res {
         Ok(timing_results) => {
@@ -113,7 +116,7 @@ fn main() {
 }
 
 /// Build the HyperfineOptions that correspond to the given ArgMatches
-fn build_hyperfine_options(matches: &ArgMatches) -> HyperfineOptions {
+fn build_hyperfine_options(matches: &ArgMatches) -> Result<HyperfineOptions, OptionsError> {
     let mut options = HyperfineOptions::default();
     let str_to_u64 = |n| u64::from_str_radix(n, 10).ok();
 
@@ -122,11 +125,41 @@ fn build_hyperfine_options(matches: &ArgMatches) -> HyperfineOptions {
         .and_then(&str_to_u64)
         .unwrap_or(0);
 
-    if let Some(min_runs) = matches.value_of("min-runs").and_then(&str_to_u64) {
-        // we need at least two runs to compute a variance
-        options.min_runs = cmp::max(2, min_runs);
+    let mut min_runs = matches.value_of("min-runs").and_then(&str_to_u64);
+    let mut max_runs = matches.value_of("max-runs").and_then(&str_to_u64);
+
+    if let Some(runs) = matches.value_of("runs").and_then(&str_to_u64) {
+        min_runs = Some(runs);
+        max_runs = Some(runs);
     }
 
+    match (min_runs, max_runs) {
+        (Some(min), _) if min < 2 => {
+            // We need at least two runs to compute a variance.
+            return Err(OptionsError::RunsBelowTwo);
+        }
+        (Some(min), None) => {
+            options.runs.min = min;
+        }
+        (_, Some(max)) if max < 2 => {
+            // We need at least two runs to compute a variance.
+            return Err(OptionsError::RunsBelowTwo);
+        }
+        (None, Some(max)) => {
+            // Since the minimum was not explicit we lower it if max is below the default min.
+            options.runs.min = cmp::min(options.runs.min, max);
+            options.runs.max = max;
+        }
+        (Some(min), Some(max)) if min > max => {
+            return Err(OptionsError::EmptyRunsRange);
+        }
+        (Some(min), Some(max)) => {
+            options.runs.min = min;
+            options.runs.max = max;
+        }
+        (None, None) => {}
+    };
+
     options.preparation_command = matches.value_of("prepare").map(String::from);
 
     options.show_output = matches.is_present("show-output");
@@ -155,7 +188,7 @@ fn build_hyperfine_options(matches: &ArgMatches) -> HyperfineOptions {
         options.failure_action = CmdFailureAction::Ignore;
     }
 
-    options
+    Ok(options)
 }
 
 /// Build the ExportManager that will export the results specified