From 7eb533f4f5563a375d2599327c47a987f3ff9d56 Mon Sep 17 00:00:00 2001 From: lukacan Date: Mon, 11 Mar 2024 22:30:44 +0100 Subject: [PATCH] Optionaly show stats during fuzzing session --- CHANGELOG.md | 1 + Cargo.lock | 55 ++++- crates/client/Cargo.toml | 1 + .../derive/fuzz_test_executor/src/lib.rs | 44 ++-- crates/client/src/commander.rs | 203 +++++++++++++----- crates/client/src/config.rs | 143 +++++++++++- crates/client/src/fuzzer/fuzzing_stats.rs | 91 ++++++++ crates/client/src/fuzzer/mod.rs | 1 + crates/client/src/lib.rs | 1 + crates/client/src/templates/Trident.toml.tmpl | 3 + crates/client/src/test_generator.rs | 2 +- .../arbitrary-custom-types-4/Cargo.lock | 67 +++++- .../arbitrary-custom-types-4/Trident.toml | 3 + .../arbitrary-limit-inputs-5/Trident.toml | 3 + examples/fuzz-tests/hello_world/Trident.toml | 3 + .../Trident.toml | 3 + .../incorrect-ix-sequence-1/Trident.toml | 3 + .../unauthorized-access-2/Trident.toml | 3 + .../unchecked-arithmetic-0/Trident.toml | 3 + .../integration-tests/escrow/Trident.toml | 3 + .../integration-tests/turnstile/Trident.toml | 3 + 21 files changed, 557 insertions(+), 82 deletions(-) create mode 100644 crates/client/src/fuzzer/fuzzing_stats.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 29c9fbfc..420dabd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ incremented upon a breaking change and the patch version will be incremented for ## [Unreleased] ### Added +- feat/fuzzer-stats-logging, an optional statistics output for fuzzing session ([#144](https://github.com/Ackee-Blockchain/trident/pull/144)) - fix/allow to process duplicate transactions ([#147](https://github.com/Ackee-Blockchain/trident/pull/147)) - feat/possibility to implement custom transaction error handling ([#145](https://github.com/Ackee-Blockchain/trident/pull/145)) - feat/support of automatically obtaining fully qualified paths of Data Accounts Custom types for `accounts_snapshots.rs` ([#141](https://github.com/Ackee-Blockchain/trident/pull/141)) diff --git a/Cargo.lock b/Cargo.lock index 14b466e3..191c7b1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1140,7 +1140,7 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ - "encode_unicode", + "encode_unicode 0.3.6", "lazy_static", "libc", "unicode-width", @@ -1273,6 +1273,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.8.0" @@ -1644,6 +1665,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -3121,6 +3148,20 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prettytable" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46480520d1b77c9a3482d39939fcf96831537a250ec62d4fd8fbdf8e0302e781" +dependencies = [ + "csv", + "encode_unicode 1.0.0", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -5609,6 +5650,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -6009,6 +6061,7 @@ dependencies = [ "macrotest", "pathdiff", "pretty_assertions", + "prettytable", "proc-macro2", "quinn-proto", "quote", diff --git a/crates/client/Cargo.toml b/crates/client/Cargo.toml index a9143134..1050104b 100644 --- a/crates/client/Cargo.toml +++ b/crates/client/Cargo.toml @@ -71,3 +71,4 @@ pathdiff = "0.2.1" solana-banks-client = "<1.18" indicatif = "0.17.8" regex = "1.10.3" +prettytable = "0.10.0" diff --git a/crates/client/derive/fuzz_test_executor/src/lib.rs b/crates/client/derive/fuzz_test_executor/src/lib.rs index 22298327..890e4621 100644 --- a/crates/client/derive/fuzz_test_executor/src/lib.rs +++ b/crates/client/derive/fuzz_test_executor/src/lib.rs @@ -13,6 +13,12 @@ pub fn fuzz_test_executor(input: TokenStream) -> TokenStream { let variant_name = &variant.ident; quote! { #enum_name::#variant_name (ix) => { + #[cfg(fuzzing_with_stats)] + let mut stats_logger = FuzzingStatistics::new(); + #[cfg(fuzzing_with_stats)] + stats_logger.increase_invoked(self.to_context_string()); + + let (mut signers, metas) = ix.get_accounts(client, &mut accounts.borrow_mut()) .map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) .expect("Accounts calculation expect"); @@ -49,25 +55,31 @@ pub fn fuzz_test_executor(input: TokenStream) -> TokenStream { None => { let tx_result = client.process_transaction(transaction) .map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))); - match tx_result { - Ok(_) => { - snaphot.capture_after(client).unwrap(); - let (acc_before, acc_after) = snaphot.get_snapshot() - .map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) - .expect("Snapshot deserialization expect"); // we want to panic if we cannot unwrap to cause a crash + Ok(_) => { + #[cfg(fuzzing_with_stats)] + stats_logger.increase_successfully_invoked(self.to_context_string()); + + snaphot.capture_after(client).unwrap(); + let (acc_before, acc_after) = snaphot.get_snapshot() + .map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) + .expect("Snapshot deserialization expect"); // we want to panic if we cannot unwrap to cause a crash - if let Err(e) = ix.check(acc_before, acc_after, data).map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) { - eprintln!( - "\x1b[31mCRASH DETECTED!\x1b[0m Custom check after the {} instruction did not pass!", - self.to_context_string()); - panic!("{}", e) + #[cfg(fuzzing_with_stats)] + stats_logger.output_serialized(); + if let Err(e) = ix.check(acc_before, acc_after, data).map_err(|e| e.with_origin(Origin::Instruction(self.to_context_string()))) { + eprintln!( + "\x1b[31mCRASH DETECTED!\x1b[0m Custom check after the {} instruction did not pass!", + self.to_context_string()); + panic!("{}", e) + } + }, + Err(e) => { + #[cfg(fuzzing_with_stats)] + stats_logger.output_serialized(); + let mut raw_accounts = snaphot.get_raw_pre_ix_accounts(); + ix.tx_error_handler(e, data, &mut raw_accounts)? } - }, - Err(e) => { - let mut raw_accounts = snaphot.get_raw_pre_ix_accounts(); - ix.tx_error_handler(e, data, &mut raw_accounts)? - } } } } diff --git a/crates/client/src/commander.rs b/crates/client/src/commander.rs index 8c9a970b..2f804207 100644 --- a/crates/client/src/commander.rs +++ b/crates/client/src/commander.rs @@ -19,6 +19,8 @@ use tokio::{ }; use crate::constants::*; +use crate::fuzzing_stats::FuzzingStatistics; +use tokio::io::AsyncBufReadExt; #[derive(Error, Debug)] pub enum Error { @@ -153,7 +155,11 @@ impl Commander { // arguments so we need to parse the variable content. let hfuzz_run_args = std::env::var("HFUZZ_RUN_ARGS").unwrap_or_default(); - let fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); + let rustflags = std::env::var("RUSTFLAGS").unwrap_or_default(); + + let rustflags = config.get_rustflags_args(rustflags); + + let mut fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); // let cargo_target_dir = std::env::var("CARGO_TARGET_DIR").unwrap_or_default(); @@ -181,36 +187,34 @@ impl Commander { } } - let mut rustflags = if config.fuzz.allow_duplicate_txs { - "--cfg allow_duplicate_txs " - } else { - "" - } - .to_string(); - - rustflags.push_str(&std::env::var("RUSTFLAGS").unwrap_or_default()); - - let mut child = Command::new("cargo") - .env("HFUZZ_RUN_ARGS", fuzz_args) - .env("CARGO_TARGET_DIR", cargo_target_dir) - .env("HFUZZ_WORKSPACE", hfuzz_workspace) - .env("RUSTFLAGS", rustflags) - .arg("hfuzz") - .arg("run") - .arg(target) - .spawn()?; - - tokio::select! { - res = child.wait() => - match res { - Ok(status) => if !status.success() { - println!("Honggfuzz exited with an error!"); - }, - Err(_) => throw!(Error::FuzzingFailed), - }, - _ = signal::ctrl_c() => { - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - }, + match rustflags.contains("fuzzing_with_stats") { + true => { + // enforce keep output to be true + fuzz_args.push_str("--keep_output"); + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .stdout(Stdio::piped()) + .spawn()?; + Self::handle_child_with_stats(&mut child).await?; + } + false => { + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .spawn()?; + Self::handle_child(&mut child).await?; + } } if let Ok(crash_files) = get_crash_files(&crash_dir, &ext) { @@ -236,27 +240,54 @@ impl Commander { let hfuzz_workspace = std::env::var("HFUZZ_WORKSPACE") .unwrap_or_else(|_| config.get_env_arg("HFUZZ_WORKSPACE")); - let fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); - - let mut rustflags = if config.fuzz.allow_duplicate_txs { - "--cfg allow_duplicate_txs " - } else { - "" + let mut fuzz_args = config.get_honggfuzz_args(hfuzz_run_args); + + let rustflags = std::env::var("RUSTFLAGS").unwrap_or_default(); + + let rustflags = config.get_rustflags_args(rustflags); + + match rustflags.contains("fuzzing_with_stats") { + true => { + // enforce keep output to be true + fuzz_args.push_str("--keep_output"); + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .stdout(Stdio::piped()) + .spawn()?; + Self::handle_child_with_stats(&mut child).await?; + } + false => { + let mut child = Command::new("cargo") + .env("HFUZZ_RUN_ARGS", fuzz_args) + .env("CARGO_TARGET_DIR", cargo_target_dir) + .env("HFUZZ_WORKSPACE", hfuzz_workspace) + .env("RUSTFLAGS", rustflags) + .arg("hfuzz") + .arg("run") + .arg(target) + .spawn()?; + Self::handle_child(&mut child).await?; + } } - .to_string(); - - rustflags.push_str(&std::env::var("RUSTFLAGS").unwrap_or_default()); - - let mut child = Command::new("cargo") - .env("HFUZZ_RUN_ARGS", fuzz_args) - .env("CARGO_TARGET_DIR", cargo_target_dir) - .env("HFUZZ_WORKSPACE", hfuzz_workspace) - .env("RUSTFLAGS", rustflags) - .arg("hfuzz") - .arg("run") - .arg(target) - .spawn()?; + } + /// Manages a child process in an async context, specifically for monitoring fuzzing tasks. + /// Waits for the process to exit or a Ctrl+C signal. Prints an error message if the process + /// exits with an error, and sleeps briefly on Ctrl+C. Throws `Error::FuzzingFailed` on errors. + /// + /// # Arguments + /// * `child` - A mutable reference to a `Child` process. + /// + /// # Errors + /// * Throws `Error::FuzzingFailed` if waiting on the child process fails. + #[throws] + async fn handle_child(child: &mut Child) { tokio::select! { res = child.wait() => match res { @@ -270,6 +301,69 @@ impl Commander { }, } } + /// Asynchronously manages a child fuzzing process, collecting and logging its statistics. + /// This function spawns a new task dedicated to reading the process's standard output and logging the fuzzing statistics. + /// It waits for either the child process to exit or a Ctrl+C signal to be received. Upon process exit or Ctrl+C signal, + /// it stops the logging task and displays the collected statistics in a table format. + /// + /// The implementation ensures that the statistics logging task only stops after receiving a signal indicating the end of the fuzzing process + /// or an interrupt from the user, preventing premature termination of the logging task if scenarios where reading is faster than fuzzing, + /// which should not be common. + /// + /// # Arguments + /// * `child` - A mutable reference to a `Child` process, representing the child fuzzing process. + /// + /// # Errors + /// * `Error::FuzzingFailed` - Thrown if there's an issue with managing the child process, such as failing to wait on the child process. + #[throws] + async fn handle_child_with_stats(child: &mut Child) { + let stdout = child + .stdout + .take() + .expect("child did not have a handle to stdout"); + + let reader = tokio::io::BufReader::new(stdout); + + let fuzz_end = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let fuzz_end_clone = std::sync::Arc::clone(&fuzz_end); + + tokio::spawn(async move { + let mut stats_logger = FuzzingStatistics::new(); + + let mut lines = reader.lines(); + loop { + // Why the lock ?? + // I`m not sure what happens if the fuzzing sessions is still active + // however the reader already read all the lines. I think that would bring us + // to the scenario where this reading task ends prematurely. + if fuzz_end_clone.load(std::sync::atomic::Ordering::SeqCst) { + break; + } + while let Ok(Some(line)) = lines.next_line().await { + stats_logger.insert_serialized(&line); + } + } + stats_logger.show_table(); + }); + + tokio::select! { + res = child.wait() =>{ + fuzz_end.store(true, std::sync::atomic::Ordering::SeqCst); + match res { + Ok(status) => { + if !status.success() { + println!("Honggfuzz exited with an error!"); + } + }, + Err(_) => throw!(Error::FuzzingFailed), + } + }, + _ = signal::ctrl_c() => { + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + fuzz_end.store(true, std::sync::atomic::Ordering::SeqCst); + }, + } + } /// Runs fuzzer on the given target. #[throws] @@ -286,14 +380,9 @@ impl Commander { let cargo_target_dir = std::env::var("CARGO_TARGET_DIR") .unwrap_or_else(|_| config.get_env_arg("CARGO_TARGET_DIR")); - let mut rustflags = if config.fuzz.allow_duplicate_txs { - "--cfg allow_duplicate_txs " - } else { - "" - } - .to_string(); + let rustflags = std::env::var("RUSTFLAGS").unwrap_or_default(); - rustflags.push_str(&std::env::var("RUSTFLAGS").unwrap_or_default()); + let rustflags = config.get_rustflags_args(rustflags); // using exec rather than spawn and replacing current process to avoid unflushed terminal output after ctrl+c signal std::process::Command::new("cargo") diff --git a/crates/client/src/config.rs b/crates/client/src/config.rs index 20ece1fd..1df19f12 100644 --- a/crates/client/src/config.rs +++ b/crates/client/src/config.rs @@ -42,20 +42,44 @@ impl From<_Test> for Test { } } -#[derive(Debug, Deserialize, Clone, Default)] +#[derive(Debug, Deserialize, Clone)] +pub struct Cfg { + pub cfg_identifier: String, + pub val: bool, +} + +#[derive(Debug, Deserialize, Clone)] pub struct Fuzz { - pub allow_duplicate_txs: bool, + pub rust_flags: Vec, } #[derive(Default, Debug, Deserialize, Clone)] struct _Fuzz { #[serde(default)] pub allow_duplicate_txs: Option, + #[serde(default)] + pub fuzzing_with_stats: Option, } impl From<_Fuzz> for Fuzz { - fn from(_t: _Fuzz) -> Self { - Self { - allow_duplicate_txs: _t.allow_duplicate_txs.unwrap_or(false), - } + fn from(_f: _Fuzz) -> Self { + let mut _self = Self { rust_flags: vec![] }; + + // allow_duplicate_txs + let allow_duplicate_txs = _f.allow_duplicate_txs.unwrap_or(false); + + _self.rust_flags.push(Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: allow_duplicate_txs, + }); + + // fuzzing_with_stats + let fuzzing_with_stats = _f.fuzzing_with_stats.unwrap_or(false); + + _self.rust_flags.push(Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: fuzzing_with_stats, + }); + + _self } } #[derive(Debug, Deserialize, Clone)] @@ -367,6 +391,22 @@ impl Config { args.push(cli_input); args.join(" ") } + pub fn get_rustflags_args(&self, cli_input: String) -> String { + let mut args: Vec = self + .fuzz + .rust_flags + .iter() + .map(|arg| { + if arg.val { + format!("--cfg {}", arg.cfg_identifier) + } else { + "".to_string() + } + }) + .collect(); + args.push(cli_input); + args.join(" ") + } pub fn get_env_arg(&self, env_variable: &str) -> String { let expect = format!("{env_variable} not found"); self.honggfuzz @@ -404,6 +444,23 @@ mod tests { } } + impl Default for Fuzz { + fn default() -> Self { + let rust_flags = vec![ + Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: false, + }, + Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: false, + }, + ]; + + Self { rust_flags } + } + } + use super::*; #[test] fn test_merge_and_precedence1() { @@ -514,4 +571,78 @@ mod tests { let hfuzz_workspace = config.get_env_arg(HFUZZ_WORKSPACE_ENV); assert_eq!(hfuzz_workspace, "new_value_y"); } + + #[test] + fn test_obtain_rustflags_variable1() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz::default(), + }; + + let rustflags = config.get_rustflags_args("".to_string()); + let default_rustflags = " "; + + assert_eq!(rustflags, default_rustflags); + } + #[test] + fn test_obtain_rustflags_variable2() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz { + rust_flags: vec![Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: true, + }], + }, + }; + + let rustflags = config.get_rustflags_args("".to_string()); + let reference_rustflags = "--cfg fuzzing_with_stats "; + + assert_eq!(rustflags, reference_rustflags); + } + #[test] + fn test_obtain_rustflags_variable3() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz { + rust_flags: vec![ + Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: true, + }, + Cfg { + cfg_identifier: "fuzzing_with_stats".to_string(), + val: false, + }, + ], + }, + }; + + let rustflags = config.get_rustflags_args("".to_string()); + let reference_rustflags = "--cfg allow_duplicate_txs "; + + assert_eq!(rustflags, reference_rustflags); + } + #[test] + fn test_obtain_rustflags_variable4() { + let config = Config { + test: Test::default(), + honggfuzz: HonggFuzz::default(), + fuzz: Fuzz { + rust_flags: vec![Cfg { + cfg_identifier: "allow_duplicate_txs".to_string(), + val: true, + }], + }, + }; + + let rustflags = config.get_rustflags_args("--cfg fuzzing_with_stats".to_string()); + let reference_rustflags = "--cfg allow_duplicate_txs --cfg fuzzing_with_stats"; + + assert_eq!(rustflags, reference_rustflags); + } } diff --git a/crates/client/src/fuzzer/fuzzing_stats.rs b/crates/client/src/fuzzer/fuzzing_stats.rs new file mode 100644 index 00000000..adf1e309 --- /dev/null +++ b/crates/client/src/fuzzer/fuzzing_stats.rs @@ -0,0 +1,91 @@ +use prettytable::{row, Table}; +use std::collections::HashMap; + +/// Represents fuzzing statistics, specifically tracking the number of times +/// an instruction was invoked and successfully executed. +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct IterationStats { + pub invoked: u64, + pub successfully_invoked: u64, +} + +/// Manages and aggregates statistics for fuzzing instructions. +#[derive(Debug, Default)] +pub struct FuzzingStatistics { + pub instructions: HashMap, +} + +impl FuzzingStatistics { + /// Constructs a new, empty `FuzzingStatistics`. + pub fn new() -> Self { + let empty_instructions = HashMap::::default(); + Self { + instructions: empty_instructions, + } + } + /// Outputs the statistics as a serialized JSON string. + pub fn output_serialized(&self) { + let serialized = serde_json::to_string(&self.instructions).unwrap(); + println!("{}", serialized); + } + + /// Increments the invocation count for a given instruction. + /// # Arguments + /// * `instruction` - The instruction to increment the count for. + pub fn increase_invoked(&mut self, instruction: String) { + self.instructions + .entry(instruction) + .and_modify(|iterations_stats| iterations_stats.invoked += 1) + .or_insert(IterationStats { + invoked: 1, + successfully_invoked: 0, + }); + } + /// Increments the successful invocation count for a given instruction. + /// # Arguments + /// * `instruction` - The instruction to increment the successful count for. + pub fn increase_successfully_invoked(&mut self, instruction: String) { + self.instructions + .entry(instruction) + .and_modify(|iterations_stats| iterations_stats.successfully_invoked += 1) + .or_insert( + // this should not occure as instruction has to be invoked + // and then successfully_invoked + IterationStats { + invoked: 1, + successfully_invoked: 1, + }, + ); + } + + /// Inserts or updates instructions with statistics provided in a serialized string. + /// # Arguments + /// * `serialized_iteration` - The serialized statistics to insert or update. + pub fn insert_serialized(&mut self, serialized_iteration: &str) { + let result = serde_json::from_str::>(serialized_iteration); + + if let Ok(deserialized_instruction) = result { + for (key, value) in deserialized_instruction { + self.instructions + .entry(key) + .and_modify(|instruction_stats| { + instruction_stats.invoked += value.invoked; + instruction_stats.successfully_invoked += value.successfully_invoked; + }) + .or_insert_with(|| IterationStats { + invoked: value.invoked, + successfully_invoked: value.successfully_invoked, + }); + } + } + } + /// Displays the collected statistics in a formatted table. + pub fn show_table(&self) { + let mut table = Table::new(); + table.add_row(row!["Instruction", "Invoked", "Successfully Invoked"]); + for (instruction, stats) in &self.instructions { + table.add_row(row![instruction, stats.invoked, stats.successfully_invoked]); + } + table.printstd(); + } +} diff --git a/crates/client/src/fuzzer/mod.rs b/crates/client/src/fuzzer/mod.rs index fc1a9b39..b478cf71 100644 --- a/crates/client/src/fuzzer/mod.rs +++ b/crates/client/src/fuzzer/mod.rs @@ -1,6 +1,7 @@ pub mod accounts_storage; pub mod data_builder; pub mod fuzzer_generator; +pub mod fuzzing_stats; #[cfg(feature = "fuzzing")] pub mod program_test_client_blocking; pub mod snapshot; diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index d2fb699e..5954d6e6 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -42,6 +42,7 @@ pub mod fuzzing { pub use super::fuzzer::accounts_storage::*; pub use super::fuzzer::data_builder::build_ix_fuzz_data; pub use super::fuzzer::data_builder::*; + pub use super::fuzzing_stats::FuzzingStatistics; pub use super::fuzzer::program_test_client_blocking::ProgramTestClientBlocking; pub use super::fuzzer::snapshot::Snapshot; diff --git a/crates/client/src/templates/Trident.toml.tmpl b/crates/client/src/templates/Trident.toml.tmpl index e05d7801..a398402c 100644 --- a/crates/client/src/templates/Trident.toml.tmpl +++ b/crates/client/src/templates/Trident.toml.tmpl @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/crates/client/src/test_generator.rs b/crates/client/src/test_generator.rs index b38531bd..92899ecc 100644 --- a/crates/client/src/test_generator.rs +++ b/crates/client/src/test_generator.rs @@ -609,7 +609,7 @@ impl TestGenerator { } None => { members.push(new_member); - println!("{FINISH} [{CARGO_TOML}] with [{member}]"); + println!("{FINISH} [{CARGO_TOML}] updated with [{member}]"); fs::write(cargo, content.to_string()).await?; } }; diff --git a/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock b/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock index 51e37221..080ebe2f 100644 --- a/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock +++ b/examples/fuzz-tests/arbitrary-custom-types-4/Cargo.lock @@ -1051,7 +1051,7 @@ version = "0.15.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" dependencies = [ - "encode_unicode", + "encode_unicode 0.3.6", "lazy_static", "libc", "unicode-width", @@ -1184,6 +1184,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.8.0" @@ -1549,6 +1570,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.33" @@ -2206,6 +2233,17 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.6", + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -2949,6 +2987,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6fa0831dd7cc608c38a5e323422a0077678fa5744aa2be4ad91c4ece8eec8d5" +[[package]] +name = "prettytable" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46480520d1b77c9a3482d39939fcf96831537a250ec62d4fd8fbdf8e0302e781" +dependencies = [ + "csv", + "encode_unicode 1.0.0", + "is-terminal", + "lazy_static", + "term", + "unicode-width", +] + [[package]] name = "proc-macro-crate" version = "0.1.5" @@ -5437,6 +5489,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -5822,6 +5885,7 @@ dependencies = [ "lazy_static", "log", "pathdiff", + "prettytable", "proc-macro2", "quinn-proto", "quote", @@ -5835,6 +5899,7 @@ dependencies = [ "solana-account-decoder", "solana-banks-client", "solana-cli-output", + "solana-logger", "solana-program-runtime", "solana-program-test-anchor-fix", "solana-sdk", diff --git a/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml b/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml +++ b/examples/fuzz-tests/arbitrary-custom-types-4/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml b/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml +++ b/examples/fuzz-tests/arbitrary-limit-inputs-5/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/hello_world/Trident.toml b/examples/fuzz-tests/hello_world/Trident.toml index 6c05d2e0..93edfd43 100644 --- a/examples/fuzz-tests/hello_world/Trident.toml +++ b/examples/fuzz-tests/hello_world/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml b/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml +++ b/examples/fuzz-tests/incorrect-integer-arithmetic-3/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml b/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml +++ b/examples/fuzz-tests/incorrect-ix-sequence-1/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/unauthorized-access-2/Trident.toml b/examples/fuzz-tests/unauthorized-access-2/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/unauthorized-access-2/Trident.toml +++ b/examples/fuzz-tests/unauthorized-access-2/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml b/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml index e05d7801..a398402c 100644 --- a/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml +++ b/examples/fuzz-tests/unchecked-arithmetic-0/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/integration-tests/escrow/Trident.toml b/examples/integration-tests/escrow/Trident.toml index e05d7801..a398402c 100644 --- a/examples/integration-tests/escrow/Trident.toml +++ b/examples/integration-tests/escrow/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false diff --git a/examples/integration-tests/turnstile/Trident.toml b/examples/integration-tests/turnstile/Trident.toml index e05d7801..a398402c 100644 --- a/examples/integration-tests/turnstile/Trident.toml +++ b/examples/integration-tests/turnstile/Trident.toml @@ -36,3 +36,6 @@ save_all = false [fuzz] # Allow processing of duplicate transactions. Setting to true might speed up fuzzing but can cause false positive crashes (default: false) allow_duplicate_txs = false +# Trident will show statistics after the fuzzing session. This option forces use of honggfuzz parameter +# `keep_output` as true in order to be able to catch fuzzer stdout. (default: false) +fuzzing_with_stats = false