feat(fuzz) - add test progress (#7914)

* feat(forge) - add test progress

* Code cleanup

* Invariant progress bar cleanup

* Display number of threads and shrink run counter

* Add progress for regular fuzz tests too

* Cleanup code, use rayon collect

* Changes after review. Cleanup

* Fix clippy

crates/evm/evm/Cargo.toml

@@ -50,3 +50,4 @@ parking_lot = "0.12"
 proptest = "1"
 thiserror = "1"
 tracing = "0.1"
+indicatif = "0.17"

crates/evm/evm/src/executors/fuzz/mod.rs

@@ -14,6 +14,7 @@ use foundry_evm_fuzz::{
     BaseCounterExample, CounterExample, FuzzCase, FuzzError, FuzzFixtures, FuzzTestResult,
 };
 use foundry_evm_traces::CallTraceArena;
+use indicatif::ProgressBar;
 use proptest::test_runner::{TestCaseError, TestError, TestRunner};
 use std::{borrow::Cow, cell::RefCell};
 
@@ -59,6 +60,7 @@ impl FuzzedExecutor {
         address: Address,
         should_fail: bool,
         rd: &RevertDecoder,
+        progress: Option<&ProgressBar>,
     ) -> FuzzTestResult {
         // Stores the first Fuzzcase
         let first_case: RefCell<Option<FuzzCase>> = RefCell::default();
@@ -91,6 +93,11 @@ impl FuzzedExecutor {
         let run_result = self.runner.clone().run(&strat, |calldata| {
             let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;
 
+            // If running with progress then increment current run.
+            if let Some(progress) = progress {
+                progress.inc(1);
+            };
+
             match fuzz_res {
                 FuzzOutcome::Case(case) => {
                     let mut first_case = first_case.borrow_mut();

crates/evm/evm/src/executors/invariant/mod.rs

@@ -20,6 +20,7 @@ use foundry_evm_fuzz::{
     FuzzCase, FuzzFixtures, FuzzedCases,
 };
 use foundry_evm_traces::CallTraceArena;
+use indicatif::ProgressBar;
 use parking_lot::RwLock;
 use proptest::{
     strategy::{Strategy, ValueTree},
@@ -135,6 +136,7 @@ impl<'a> InvariantExecutor<'a> {
         &mut self,
         invariant_contract: InvariantContract<'_>,
         fuzz_fixtures: &FuzzFixtures,
+        progress: Option<&ProgressBar>,
     ) -> Result<InvariantFuzzTestResult> {
         // Throw an error to abort test run if the invariant function accepts input params
         if !invariant_contract.invariant_function.inputs.is_empty() {
@@ -314,6 +316,11 @@ impl<'a> InvariantExecutor<'a> {
             // Revert state to not persist values between runs.
             fuzz_state.revert();
 
+            // If running with progress then increment completed runs.
+            if let Some(progress) = progress {
+                progress.inc(1);
+            }
+
             Ok(())
         });
 

crates/evm/evm/src/executors/invariant/replay.rs

@@ -11,6 +11,7 @@ use foundry_evm_fuzz::{
     BaseCounterExample,
 };
 use foundry_evm_traces::{load_contracts, TraceKind, Traces};
+use indicatif::ProgressBar;
 use parking_lot::RwLock;
 use proptest::test_runner::TestError;
 use revm::primitives::U256;
@@ -97,13 +98,14 @@ pub fn replay_error(
     logs: &mut Vec<Log>,
     traces: &mut Traces,
     coverage: &mut Option<HitMaps>,
+    progress: Option<&ProgressBar>,
 ) -> Result<Vec<BaseCounterExample>> {
     match failed_case.test_error {
         // Don't use at the moment.
         TestError::Abort(_) => Ok(vec![]),
         TestError::Fail(_, ref calls) => {
             // Shrink sequence of failed calls.
-            let calls = shrink_sequence(failed_case, calls, &executor)?;
+            let calls = shrink_sequence(failed_case, calls, &executor, progress)?;
 
             set_up_inner_replay(&mut executor, &failed_case.inner_sequence);
 

crates/evm/evm/src/executors/invariant/shrink.rs

@@ -2,8 +2,9 @@ use crate::executors::{invariant::error::FailedInvariantCaseData, Executor};
 use alloy_primitives::{Address, Bytes, U256};
 use foundry_evm_core::constants::CALLER;
 use foundry_evm_fuzz::invariant::BasicTxDetails;
+use indicatif::ProgressBar;
 use proptest::bits::{BitSetLike, VarBitSet};
-use std::borrow::Cow;
+use std::{borrow::Cow, cmp::min};
 
 #[derive(Clone, Copy, Debug)]
 struct Shrink {
@@ -83,9 +84,17 @@ pub(crate) fn shrink_sequence(
     failed_case: &FailedInvariantCaseData,
     calls: &[BasicTxDetails],
     executor: &Executor,
+    progress: Option<&ProgressBar>,
 ) -> eyre::Result<Vec<BasicTxDetails>> {
     trace!(target: "forge::test", "Shrinking sequence of {} calls.", calls.len());
 
+    // Reset run count and display shrinking message.
+    if let Some(progress) = progress {
+        progress.set_length(min(calls.len(), failed_case.shrink_run_limit as usize) as u64);
+        progress.reset();
+        progress.set_message(" Shrink");
+    }
+
     // Special case test: the invariant is *unsatisfiable* - it took 0 calls to
     // break the invariant -- consider emitting a warning.
     let error_call_result =
@@ -112,6 +121,10 @@ pub(crate) fn shrink_sequence(
             Ok((true, _)) if !shrinker.complicate() => break,
             _ => {}
         }
+
+        if let Some(progress) = progress {
+            progress.inc(1);
+        }
     }
 
     Ok(shrinker.current().map(|idx| &calls[idx]).cloned().collect())

crates/forge/bin/cmd/test/mod.rs

@@ -135,6 +135,10 @@ pub struct TestArgs {
     /// Print detailed test summary table.
     #[arg(long, help_heading = "Display options", requires = "summary")]
     pub detailed: bool,
+
+    /// Show test execution progress.
+    #[arg(long)]
+    pub show_progress: bool,
 }
 
 impl TestArgs {
@@ -387,9 +391,10 @@ impl TestArgs {
         // Run tests.
         let (tx, rx) = channel::<(String, SuiteResult)>();
         let timer = Instant::now();
+        let show_progress = self.show_progress;
         let handle = tokio::task::spawn_blocking({
             let filter = filter.clone();
-            move || runner.test(&filter, tx)
+            move || runner.test(&filter, tx, show_progress)
         });
 
         // Set up trace identifiers.

crates/forge/src/lib.rs

@@ -24,6 +24,7 @@ pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder};
 mod runner;
 pub use runner::ContractRunner;
 
+mod progress;
 pub mod result;
 
 // TODO: remove

crates/forge/src/multi_runner.rs

@@ -1,7 +1,8 @@
 //! Forge test runner for multiple contracts.
 
 use crate::{
-    result::SuiteResult, runner::LIBRARY_DEPLOYER, ContractRunner, TestFilter, TestOptions,
+    progress::TestsProgress, result::SuiteResult, runner::LIBRARY_DEPLOYER, ContractRunner,
+    TestFilter, TestOptions,
 };
 use alloy_json_abi::{Function, JsonAbi};
 use alloy_primitives::{Address, Bytes, U256};
@@ -140,7 +141,7 @@ impl MultiContractRunner {
         filter: &dyn TestFilter,
     ) -> impl Iterator<Item = (String, SuiteResult)> {
         let (tx, rx) = mpsc::channel();
-        self.test(filter, tx);
+        self.test(filter, tx, false);
         rx.into_iter()
     }
 
@@ -150,7 +151,12 @@ impl MultiContractRunner {
     /// before executing all contracts and their tests in _parallel_.
     ///
     /// Each Executor gets its own instance of the `Backend`.
-    pub fn test(&mut self, filter: &dyn TestFilter, tx: mpsc::Sender<(String, SuiteResult)>) {
+    pub fn test(
+        &mut self,
+        filter: &dyn TestFilter,
+        tx: mpsc::Sender<(String, SuiteResult)>,
+        show_progress: bool,
+    ) {
         let handle = tokio::runtime::Handle::current();
         trace!("running all tests");
 
@@ -167,11 +173,45 @@ impl MultiContractRunner {
             find_time,
         );
 
-        contracts.par_iter().for_each(|&(id, contract)| {
-            let _guard = handle.enter();
-            let result = self.run_tests(id, contract, db.clone(), filter, &handle);
-            let _ = tx.send((id.identifier(), result));
-        })
+        if show_progress {
+            let tests_progress = TestsProgress::new(contracts.len(), rayon::current_num_threads());
+            // Collect test suite results to stream at the end of test run.
+            let results: Vec<(String, SuiteResult)> = contracts
+                .par_iter()
+                .map(|&(id, contract)| {
+                    let _guard = handle.enter();
+                    tests_progress.inner.lock().start_suite_progress(&id.identifier());
+
+                    let result = self.run_tests(
+                        id,
+                        contract,
+                        db.clone(),
+                        filter,
+                        &handle,
+                        Some(&tests_progress),
+                    );
+
+                    tests_progress
+                        .inner
+                        .lock()
+                        .end_suite_progress(&id.identifier(), result.summary());
+
+                    (id.identifier(), result)
+                })
+                .collect();
+
+            tests_progress.inner.lock().clear();
+
+            results.iter().for_each(|result| {
+                let _ = tx.send(result.to_owned());
+            });
+        } else {
+            contracts.par_iter().for_each(|&(id, contract)| {
+                let _guard = handle.enter();
+                let result = self.run_tests(id, contract, db.clone(), filter, &handle, None);
+                let _ = tx.send((id.identifier(), result));
+            })
+        }
     }
 
     fn run_tests(
@@ -181,6 +221,7 @@ impl MultiContractRunner {
         db: Backend,
         filter: &dyn TestFilter,
         handle: &tokio::runtime::Handle,
+        progress: Option<&TestsProgress>,
     ) -> SuiteResult {
         let identifier = artifact_id.identifier();
         let mut span_name = identifier.as_str();
@@ -222,7 +263,9 @@ impl MultiContractRunner {
             self.sender,
             &self.revert_decoder,
             self.debug,
+            progress,
         );
+
         let r = runner.run_tests(filter, &self.test_options, self.known_contracts.clone(), handle);
 
         debug!(duration=?r.duration, "executed all tests in contract");

crates/forge/src/progress.rs

@@ -0,0 +1,116 @@
+use indicatif::{MultiProgress, ProgressBar};
+use parking_lot::Mutex;
+use std::{collections::HashMap, sync::Arc, time::Duration};
+
+/// State of [ProgressBar]s displayed for the given test run.
+/// Shows progress of all test suites matching filter.
+/// For each test within the test suite an individual progress bar is displayed.
+/// When a test suite completes, their progress is removed from overall progress and result summary
+/// is displayed.
+#[derive(Debug)]
+pub struct TestsProgressState {
+    /// Main [MultiProgress] instance showing progress for all test suites.
+    multi: MultiProgress,
+    /// Progress bar counting completed / remaining test suites.
+    overall_progress: ProgressBar,
+    /// Individual test suites progress.
+    suites_progress: HashMap<String, ProgressBar>,
+}
+
+impl TestsProgressState {
+    // Creates overall tests progress state.
+    pub fn new(suites_len: usize, threads_no: usize) -> Self {
+        let multi = MultiProgress::new();
+        let overall_progress = multi.add(ProgressBar::new(suites_len as u64));
+        overall_progress.set_style(
+            indicatif::ProgressStyle::with_template("{bar:40.cyan/blue} {pos:>7}/{len:7} {msg}")
+                .unwrap()
+                .progress_chars("##-"),
+        );
+        overall_progress.set_message(format!("completed (with {} threads)", threads_no as u64));
+        Self { multi, overall_progress, suites_progress: HashMap::default() }
+    }
+
+    /// Creates new test suite progress and add it to overall progress.
+    pub fn start_suite_progress(&mut self, suite_name: &String) {
+        let suite_progress = self.multi.add(ProgressBar::new_spinner());
+        suite_progress.set_style(
+            indicatif::ProgressStyle::with_template("{spinner} {wide_msg:.bold.dim}")
+                .unwrap()
+                .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "),
+        );
+        suite_progress.set_message(format!("{suite_name} "));
+        suite_progress.enable_steady_tick(Duration::from_millis(100));
+        self.suites_progress.insert(suite_name.to_owned(), suite_progress);
+    }
+
+    /// Prints suite result summary and removes it from overall progress.
+    pub fn end_suite_progress(&mut self, suite_name: &String, result_summary: String) {
+        if let Some(suite_progress) = self.suites_progress.remove(suite_name) {
+            self.multi.suspend(|| {
+                println!("{suite_name}\n  ↪ {result_summary}");
+            });
+            suite_progress.finish_and_clear();
+            // Increment test progress bar to reflect completed test suite.
+            self.overall_progress.inc(1);
+        }
+    }
+
+    /// Creates progress entry for fuzz tests.
+    /// Set the prefix and total number of runs. Message is updated during execution with current
+    /// phase. Test progress is placed under test suite progress entry so all tests within suite
+    /// are grouped.
+    pub fn start_fuzz_progress(
+        &mut self,
+        suite_name: &str,
+        test_name: &String,
+        runs: u32,
+    ) -> Option<ProgressBar> {
+        if let Some(suite_progress) = self.suites_progress.get(suite_name) {
+            let fuzz_progress =
+                self.multi.insert_after(suite_progress, ProgressBar::new(runs as u64));
+            fuzz_progress.set_style(
+                indicatif::ProgressStyle::with_template(
+                    "    ↪ {prefix:.bold.dim}: [{pos}/{len}]{msg} Runs",
+                )
+                .unwrap()
+                .tick_chars("⠁⠂⠄⡀⢀⠠⠐⠈ "),
+            );
+            fuzz_progress.set_prefix(test_name.to_string());
+            Some(fuzz_progress)
+        } else {
+            None
+        }
+    }
+
+    /// Removes overall test progress.
+    pub fn clear(&mut self) {
+        self.multi.clear().unwrap();
+    }
+}
+
+/// Clonable wrapper around [TestsProgressState].
+#[derive(Debug, Clone)]
+pub struct TestsProgress {
+    pub inner: Arc<Mutex<TestsProgressState>>,
+}
+
+impl TestsProgress {
+    pub fn new(suites_len: usize, threads_no: usize) -> Self {
+        Self { inner: Arc::new(Mutex::new(TestsProgressState::new(suites_len, threads_no))) }
+    }
+}
+
+/// Helper function for creating fuzz test progress bar.
+pub fn start_fuzz_progress(
+    tests_progress: Option<&TestsProgress>,
+    suite_name: &str,
+    test_name: &String,
+    runs: u32,
+) -> Option<ProgressBar> {
+    if let Some(progress) = tests_progress {
+        progress.inner.lock().start_fuzz_progress(suite_name, test_name, runs)
+    } else {
+        None
+    }
+}