Skip to content

Commit

Permalink
Add progress for regular fuzz tests too
Browse files Browse the repository at this point in the history
  • Loading branch information
grandizzy committed May 17, 2024
1 parent 0672dff commit 0a338f2
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 35 deletions.
31 changes: 17 additions & 14 deletions crates/cli/src/utils/cmd.rs
Original file line number Diff line number Diff line change
Expand Up @@ -198,23 +198,26 @@ macro_rules! init_test_suite_progress {
}};
}

/// Creates progress entry for invariant tests.
/// Creates progress entry for fuzz tests.
/// Set the prefix and total number of runs. Message is updated during execution with current phase.
/// Test progress is placed under test suite progress entry so all tests within suite are grouped.
#[macro_export]
macro_rules! init_invariant_test_progress {
($overall_progress:expr, $suite_progress:expr, $test_name:expr, $runs:expr) => {{
let pb = $overall_progress
.insert_after($suite_progress, indicatif::ProgressBar::new($runs as u64));
pb.set_style(
indicatif::ProgressStyle::with_template(
" ↪ {prefix:.bold.dim}: [{pos}/{len}] {wide_msg}",
)
.unwrap()
.tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "),
);
pb.set_prefix(format!("{}", $test_name));
pb
macro_rules! init_fuzz_test_progress {
($progress:expr, $test_name:expr, $runs:expr) => {{
let test_progress = $progress.map(|(overall_progress, suite_progress)| {
let pb = overall_progress
.insert_after(suite_progress, indicatif::ProgressBar::new($runs as u64));
pb.set_style(
indicatif::ProgressStyle::with_template(
" ↪ {prefix:.bold.dim}: [{pos}/{len}]{msg} Runs",
)
.unwrap()
.tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "),
);
pb.set_prefix(format!("{}", $test_name));
pb
});
test_progress
}};
}

Expand Down
7 changes: 7 additions & 0 deletions crates/evm/evm/src/executors/fuzz/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ use foundry_evm_fuzz::{
BaseCounterExample, CounterExample, FuzzCase, FuzzError, FuzzFixtures, FuzzTestResult,
};
use foundry_evm_traces::CallTraceArena;
use indicatif::ProgressBar;
use proptest::test_runner::{TestCaseError, TestError, TestRunner};
use std::{borrow::Cow, cell::RefCell};

Expand Down Expand Up @@ -59,6 +60,7 @@ impl FuzzedExecutor {
address: Address,
should_fail: bool,
rd: &RevertDecoder,
progress: Option<&ProgressBar>,
) -> FuzzTestResult {
// Stores the first Fuzzcase
let first_case: RefCell<Option<FuzzCase>> = RefCell::default();
Expand Down Expand Up @@ -91,6 +93,11 @@ impl FuzzedExecutor {
let run_result = self.runner.clone().run(&strat, |calldata| {
let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;

// If running with progress then increment current run.
if let Some(progress) = progress {
progress.inc(1);
};

match fuzz_res {
FuzzOutcome::Case(case) => {
let mut first_case = first_case.borrow_mut();
Expand Down
5 changes: 0 additions & 5 deletions crates/evm/evm/src/executors/invariant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -178,11 +178,6 @@ impl<'a> InvariantExecutor<'a> {
fuzz_cases.borrow_mut().push(FuzzedCases::new(vec![]));
}

// If running with progress then set run label.
if let Some(progress) = progress {
progress.set_message("Runs");
}

// The strategy only comes with the first `input`. We fill the rest of the `inputs`
// until the desired `depth` so we can use the evolving fuzz dictionary
// during the run. We need another proptest runner to query for random
Expand Down
2 changes: 1 addition & 1 deletion crates/evm/evm/src/executors/invariant/shrink.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ pub(crate) fn shrink_sequence(
if let Some(progress) = progress {
progress.set_length(min(calls.len(), failed_case.shrink_run_limit) as u64);
progress.reset();
progress.set_message("Shrink Runs");
progress.set_message(" Shrink");
}

// Special case test: the invariant is *unsatisfiable* - it took 0 calls to
Expand Down
34 changes: 19 additions & 15 deletions crates/forge/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use alloy_dyn_abi::DynSolValue;
use alloy_json_abi::Function;
use alloy_primitives::{Address, U256};
use eyre::Result;
use foundry_cli::init_invariant_test_progress;
use foundry_cli::init_fuzz_test_progress;
use foundry_common::{
contracts::{ContractsByAddress, ContractsByArtifact},
TestFunctionExt,
Expand Down Expand Up @@ -359,30 +359,27 @@ impl<'a> ContractRunner<'a> {
let res = if func.is_invariant_test() {
let runner = test_options.invariant_runner(self.name, &func.name);
let invariant_config = test_options.invariant_config(self.name, &func.name);

let test_progress = progress.map(|(overall_progress, suite_progress)| {
init_invariant_test_progress!(
overall_progress,
suite_progress,
func.name,
invariant_config.runs
)
});

self.run_invariant_test(
runner,
setup,
*invariant_config,
func,
&known_contracts,
identified_contracts.as_ref().unwrap(),
test_progress,
init_fuzz_test_progress!(progress, func.name, invariant_config.runs),
)
} else if func.is_fuzz_test() {
debug_assert!(func.is_test());
let runner = test_options.fuzz_runner(self.name, &func.name);
let fuzz_config = test_options.fuzz_config(self.name, &func.name);
self.run_fuzz_test(func, should_fail, runner, setup, fuzz_config.clone())
self.run_fuzz_test(
func,
should_fail,
runner,
setup,
fuzz_config.clone(),
init_fuzz_test_progress!(progress, func.name, fuzz_config.runs),
)
} else {
debug_assert!(func.is_test());
self.run_test(func, should_fail, setup)
Expand Down Expand Up @@ -674,6 +671,7 @@ impl<'a> ContractRunner<'a> {
runner: TestRunner,
setup: TestSetup,
fuzz_config: FuzzConfig,
progress: Option<ProgressBar>,
) -> TestResult {
let span = info_span!("fuzz_test", %should_fail);
if !span.is_disabled() {
Expand Down Expand Up @@ -704,8 +702,14 @@ impl<'a> ContractRunner<'a> {
self.sender,
fuzz_config.clone(),
);
let result =
fuzzed_executor.fuzz(func, &fuzz_fixtures, address, should_fail, self.revert_decoder);
let result = fuzzed_executor.fuzz(
func,
&fuzz_fixtures,
address,
should_fail,
self.revert_decoder,
progress.as_ref(),
);

let mut debug = Default::default();
let mut breakpoints = Default::default();
Expand Down

0 comments on commit 0a338f2

Please sign in to comment.