From 552d2928cade50d803b33c8da7d0a47427e1fbab Mon Sep 17 00:00:00 2001 From: bbarwik Date: Tue, 31 Oct 2023 19:00:15 +0000 Subject: [PATCH 1/3] Added initial coverage support (scrypto coverage command) --- assets/template/.gitignore | 1 + radix-engine-common/Cargo.toml | 2 + .../src/constants/transaction_execution.rs | 6 + radix-engine-common/src/constants/wasm.rs | 3 + radix-engine/Cargo.toml | 3 + .../src/transaction/transaction_executor.rs | 14 ++ radix-engine/src/utils/coverage.rs | 29 +++ radix-engine/src/utils/mod.rs | 4 + radix-engine/src/vm/wasm/prepare.rs | 21 +- radix-engine/src/vm/wasm/wasmer.rs | 36 +++- radix-engine/src/vm/wasm/wasmi.rs | 48 ++++- scrypto-unit/Cargo.toml | 1 + scrypto-unit/src/test_runner.rs | 97 ++++++--- scrypto/Cargo.toml | 4 + scrypto/src/lib.rs | 9 + simulator/Cargo.lock | 1 + simulator/Cargo.toml | 1 + simulator/src/resim/cmd_publish.rs | 1 + simulator/src/scrypto/cmd_build.rs | 1 + simulator/src/scrypto/cmd_coverage.rs | 204 ++++++++++++++++++ simulator/src/scrypto/cmd_test.rs | 1 + simulator/src/scrypto/error.rs | 2 + simulator/src/scrypto/mod.rs | 4 + simulator/src/scrypto_bindgen/mod.rs | 5 +- simulator/src/utils/cargo.rs | 46 +++- simulator/src/utils/coverage.rs | 9 + simulator/src/utils/mod.rs | 2 + 27 files changed, 497 insertions(+), 58 deletions(-) create mode 100644 radix-engine/src/utils/coverage.rs create mode 100644 simulator/src/scrypto/cmd_coverage.rs create mode 100644 simulator/src/utils/coverage.rs diff --git a/assets/template/.gitignore b/assets/template/.gitignore index ea8c4bf7f35..e14541ff209 100644 --- a/assets/template/.gitignore +++ b/assets/template/.gitignore @@ -1 +1,2 @@ /target +/coverage diff --git a/radix-engine-common/Cargo.toml b/radix-engine-common/Cargo.toml index e25471c83e4..ae701cc3599 100644 --- a/radix-engine-common/Cargo.toml +++ b/radix-engine-common/Cargo.toml @@ -51,6 +51,8 @@ radix_engine_fuzzing = ["arbitrary", "serde", "bnum/arbitrary", "bnum/serde", "s resource_tracker=[] full_math_benches = [ "dep:rug", "dep:ethnum"] +coverage = [] + # Ref: https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options [lib] doctest = false diff --git a/radix-engine-common/src/constants/transaction_execution.rs b/radix-engine-common/src/constants/transaction_execution.rs index 8f57462174a..b6c6c8610c9 100644 --- a/radix-engine-common/src/constants/transaction_execution.rs +++ b/radix-engine-common/src/constants/transaction_execution.rs @@ -26,10 +26,16 @@ pub const MAX_TRACK_SUBSTATE_TOTAL_BYTES: usize = 64 * 1024 * 1024; pub const MAX_SUBSTATE_KEY_SIZE: usize = 1024; /// The maximum substate read and write size. +#[cfg(not(feature = "coverage"))] pub const MAX_SUBSTATE_VALUE_SIZE: usize = 2 * 1024 * 1024; +#[cfg(feature = "coverage")] +pub const MAX_SUBSTATE_VALUE_SIZE: usize = 64 * 1024 * 1024; /// The maximum invoke payload size. +#[cfg(not(feature = "coverage"))] pub const MAX_INVOKE_PAYLOAD_SIZE: usize = 1 * 1024 * 1024; +#[cfg(feature = "coverage")] +pub const MAX_INVOKE_PAYLOAD_SIZE: usize = 32 * 1024 * 1024; /// The proposer's share of tips pub const TIPS_PROPOSER_SHARE_PERCENTAGE: u8 = 100; diff --git a/radix-engine-common/src/constants/wasm.rs b/radix-engine-common/src/constants/wasm.rs index 29e16590286..9c073bfbad4 100644 --- a/radix-engine-common/src/constants/wasm.rs +++ b/radix-engine-common/src/constants/wasm.rs @@ -1,5 +1,8 @@ /// The maximum memory size (per call frame): 64 * 64KiB = 4MiB +#[cfg(not(feature = "coverage"))] pub const MAX_MEMORY_SIZE_IN_PAGES: u32 = 64; +#[cfg(feature = "coverage")] +pub const MAX_MEMORY_SIZE_IN_PAGES: u32 = 512; /// The maximum initial table size pub const MAX_INITIAL_TABLE_SIZE: u32 = 1024; diff --git a/radix-engine/Cargo.toml b/radix-engine/Cargo.toml index 9b7ad940b29..2614f8638c9 100644 --- a/radix-engine/Cargo.toml +++ b/radix-engine/Cargo.toml @@ -95,6 +95,9 @@ radix_engine_tests = [] full_wasm_benchmarks = [] +# This flag disables package size limit, memory size limit and fee limit +coverage = [ "radix-engine-common/coverage" ] + # Ref: https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options [lib] doctest = false diff --git a/radix-engine/src/transaction/transaction_executor.rs b/radix-engine/src/transaction/transaction_executor.rs index 4dee8ed254a..136f13ab8d5 100644 --- a/radix-engine/src/transaction/transaction_executor.rs +++ b/radix-engine/src/transaction/transaction_executor.rs @@ -59,6 +59,7 @@ pub struct CostingParameters { } impl Default for CostingParameters { + #[cfg(not(feature = "coverage"))] fn default() -> Self { Self { execution_cost_unit_price: EXECUTION_COST_UNIT_PRICE_IN_XRD.try_into().unwrap(), @@ -71,6 +72,19 @@ impl Default for CostingParameters { archive_storage_price: ARCHIVE_STORAGE_PRICE_IN_XRD.try_into().unwrap(), } } + #[cfg(feature = "coverage")] + fn default() -> Self { + Self { + execution_cost_unit_price: Decimal::zero(), + execution_cost_unit_limit: u32::MAX, + execution_cost_unit_loan: u32::MAX, + finalization_cost_unit_price: Decimal::zero(), + finalization_cost_unit_limit: u32::MAX, + usd_price: USD_PRICE_IN_XRD.try_into().unwrap(), + state_storage_price: Decimal::zero(), + archive_storage_price: Decimal::zero(), + } + } } impl CostingParameters { diff --git a/radix-engine/src/utils/coverage.rs b/radix-engine/src/utils/coverage.rs new file mode 100644 index 00000000000..91a1eed7b5f --- /dev/null +++ b/radix-engine/src/utils/coverage.rs @@ -0,0 +1,29 @@ +use crate::types::*; +use std::env; +use std::fs::{self, File}; +use std::io::Write; +use std::path::Path; + +#[cfg(feature = "coverage")] +pub fn save_coverage_data(blueprint_name: &String, coverage_data: &Vec) { + if let Some(dir) = env::var_os("COVERAGE_DIRECTORY") { + let mut file_path = Path::new(&dir).to_path_buf(); + file_path.push(blueprint_name); + + // Check if the blueprint directory exists, if not create it + if !file_path.exists() { + // error is ignored because when multiple tests are running it may fail + fs::create_dir(&file_path).ok(); + } + + // Write .profraw binary data + let file_name = hash(&coverage_data); + let file_name: String = file_name.0[..16] + .iter() + .map(|byte| format!("{:02x}", byte)) + .collect(); + file_path.push(format!("{}.profraw", file_name)); + let mut file = File::create(file_path).unwrap(); + file.write_all(&coverage_data).unwrap(); + } +} diff --git a/radix-engine/src/utils/mod.rs b/radix-engine/src/utils/mod.rs index 37e04d6ef18..0e1301e78cb 100644 --- a/radix-engine/src/utils/mod.rs +++ b/radix-engine/src/utils/mod.rs @@ -1,8 +1,12 @@ +#[cfg(feature = "coverage")] +mod coverage; mod macros; mod native_blueprint_call_validator; mod package_extractor; mod panics; +#[cfg(feature = "coverage")] +pub use coverage::*; pub use macros::*; pub use native_blueprint_call_validator::*; pub use package_extractor::*; diff --git a/radix-engine/src/vm/wasm/prepare.rs b/radix-engine/src/vm/wasm/prepare.rs index b5020978fbc..b10036aa1bc 100644 --- a/radix-engine/src/vm/wasm/prepare.rs +++ b/radix-engine/src/vm/wasm/prepare.rs @@ -983,15 +983,18 @@ impl WasmModule { mut self, rules: &R, ) -> Result { - let backend = gas_metering::host_function::Injector::new( - MODULE_ENV_NAME, - COSTING_CONSUME_WASM_EXECUTION_UNITS_FUNCTION_NAME, - ); - gas_metering::inject(&mut self.module, backend, rules).map_err(|err| { - PrepareError::RejectedByInstructionMetering { - reason: err.to_string(), - } - })?; + #[cfg(not(feature = "coverage"))] + { + let backend = gas_metering::host_function::Injector::new( + MODULE_ENV_NAME, + COSTING_CONSUME_WASM_EXECUTION_UNITS_FUNCTION_NAME, + ); + gas_metering::inject(&mut self.module, backend, rules).map_err(|err| { + PrepareError::RejectedByInstructionMetering { + reason: err.to_string(), + } + })?; + } Ok(self) } diff --git a/radix-engine/src/vm/wasm/wasmer.rs b/radix-engine/src/vm/wasm/wasmer.rs index 07aab72bd47..faf5e62c704 100644 --- a/radix-engine/src/vm/wasm/wasmer.rs +++ b/radix-engine/src/vm/wasm/wasmer.rs @@ -1,5 +1,7 @@ use crate::errors::InvokeError; use crate::types::*; +#[cfg(feature = "coverage")] +use crate::utils::save_coverage_data; use crate::vm::wasm::constants::*; use crate::vm::wasm::errors::*; use crate::vm::wasm::traits::*; @@ -835,13 +837,37 @@ impl WasmInstance for WasmerInstance { .map_err(|e| { let err: InvokeError = e.into(); err - })?; + }); + + let result = match return_data { + Ok(data) => { + if let Some(v) = data.as_ref().get(0).and_then(|x| x.i64()) { + read_slice(&self.instance, Slice::transmute_i64(v)).map_err(InvokeError::SelfError) + } else { + Err(InvokeError::SelfError(WasmRuntimeError::InvalidWasmPointer)) + } + } + Err(err) => Err(err) + }; - if let Some(v) = return_data.as_ref().get(0).and_then(|x| x.i64()) { - read_slice(&self.instance, Slice::transmute_i64(v)).map_err(InvokeError::SelfError) - } else { - Err(InvokeError::SelfError(WasmRuntimeError::InvalidWasmPointer)) + #[cfg(feature = "coverage")] + if let Ok(dump_coverage) = self.instance.exports.get_function("dump_coverage") { + if let Ok(blueprint_buffer) = runtime.actor_get_blueprint_name() { + let blueprint_name = + String::from_utf8(runtime.buffer_consume(blueprint_buffer.id()).unwrap()) + .unwrap(); + + let mut ret = dump_coverage.call(&[]).unwrap().as_ref().get(0).and_then(|x| x.i64()).unwrap(); + let coverage_data = read_slice( + &self.instance, + Slice::transmute_i64(ret), + ) + .unwrap(); + save_coverage_data(&blueprint_name, &coverage_data); + } } + + result } } diff --git a/radix-engine/src/vm/wasm/wasmi.rs b/radix-engine/src/vm/wasm/wasmi.rs index 641084cfb01..5d2805ed4d3 100644 --- a/radix-engine/src/vm/wasm/wasmi.rs +++ b/radix-engine/src/vm/wasm/wasmi.rs @@ -11,6 +11,8 @@ use wasmi::*; use crate::errors::InvokeError; use crate::types::*; +#[cfg(feature = "coverage")] +use crate::utils::save_coverage_data; use crate::vm::wasm::constants::*; use crate::vm::wasm::errors::*; use crate::vm::wasm::traits::*; @@ -1537,21 +1539,47 @@ impl WasmInstance for WasmiInstance { .collect(); let mut ret = [Value::I64(0)]; - let _result = func + let call_result = func .call(self.store.as_context_mut(), &input, &mut ret) .map_err(|e| { let err: InvokeError = e.into(); err - })?; - - match i64::try_from(ret[0]) { - Ok(ret) => read_slice( - self.store.as_context_mut(), - self.memory, - Slice::transmute_i64(ret), - ), - _ => Err(InvokeError::SelfError(WasmRuntimeError::InvalidWasmPointer)), + }); + + let result = match call_result { + Ok(_) => match i64::try_from(ret[0]) { + Ok(ret) => read_slice( + self.store.as_context_mut(), + self.memory, + Slice::transmute_i64(ret), + ), + _ => Err(InvokeError::SelfError(WasmRuntimeError::InvalidWasmPointer)), + }, + Err(err) => Err(err), + }; + + #[cfg(feature = "coverage")] + if let Ok(dump_coverage) = self.get_export_func("dump_coverage") { + if let Ok(blueprint_buffer) = runtime.actor_get_blueprint_name() { + let blueprint_name = + String::from_utf8(runtime.buffer_consume(blueprint_buffer.id()).unwrap()) + .unwrap(); + + let mut ret = [Value::I64(0)]; + dump_coverage + .call(self.store.as_context_mut(), &[], &mut ret) + .unwrap(); + let coverage_data = read_slice( + self.store.as_context_mut(), + self.memory, + Slice::transmute_i64(i64::try_from(ret[0]).unwrap()), + ) + .unwrap(); + save_coverage_data(&blueprint_name, &coverage_data); + } } + + result } } diff --git a/scrypto-unit/Cargo.toml b/scrypto-unit/Cargo.toml index a9232c9cbeb..1bd0bb2aa07 100644 --- a/scrypto-unit/Cargo.toml +++ b/scrypto-unit/Cargo.toml @@ -28,6 +28,7 @@ lru = ["radix-engine/lru", "radix-engine-queries/lru"] rocksdb = ["radix-engine-stores/rocksdb"] post_run_db_check = [] +coverage = ["radix-engine/coverage"] [lib] doctest = false diff --git a/scrypto-unit/src/test_runner.rs b/scrypto-unit/src/test_runner.rs index 4f4c3e8471b..36b9960ede8 100644 --- a/scrypto-unit/src/test_runner.rs +++ b/scrypto-unit/src/test_runner.rs @@ -79,31 +79,7 @@ impl Compile { package_dir: P, env_vars: sbor::rust::collections::BTreeMap, ) -> (Vec, PackageDefinition) { - // Build - let status = Command::new("cargo") - .envs(env_vars) - .current_dir(package_dir.as_ref()) - .args([ - "build", - "--target", - "wasm32-unknown-unknown", - "--release", - "--features", - "scrypto/log-error,scrypto/log-warn,scrypto/log-info,scrypto/log-debug,scrypto/log-trace" - ]) - .status() - .unwrap_or_else(|error| { - panic!( - "Compiling \"{:?}\" failed with \"{:?}\"", - package_dir.as_ref(), - error - ) - }); - if !status.success() { - panic!("Failed to compile package: {:?}", package_dir.as_ref()); - } - - // Find wasm path + // Find wasm name let mut cargo = package_dir.as_ref().to_owned(); cargo.push("Cargo.toml"); let wasm_name = if cargo.exists() { @@ -122,12 +98,79 @@ impl Compile { .to_owned() .replace("-", "_") }; - let mut path = PathBuf::from_str(&get_cargo_target_directory(&cargo)).unwrap(); // Infallible; + + let mut path = PathBuf::from_str(&get_cargo_target_directory(&cargo)).unwrap(); path.push("wasm32-unknown-unknown"); path.push("release"); - path.push(wasm_name); + path.push(&wasm_name); path.set_extension("wasm"); + #[cfg(feature = "coverage")] + // Check if binary exists in coverage directory, if it doesn't only then build it + { + let mut coverage_path = PathBuf::from_str(&get_cargo_target_directory(&cargo)).unwrap(); + coverage_path.pop(); + coverage_path.push("coverage"); + coverage_path.push("wasm32-unknown-unknown"); + coverage_path.push("release"); + coverage_path.push(wasm_name); + coverage_path.set_extension("wasm"); + if coverage_path.is_file() { + let code = fs::read(&coverage_path).unwrap_or_else(|err| { + panic!( + "Failed to read built WASM from path {:?} - {:?}", + &path, err + ) + }); + coverage_path.set_extension("rpd"); + let definition = fs::read(&coverage_path).unwrap_or_else(|err| { + panic!( + "Failed to read manifest definition from path {:?} - {:?}", + &coverage_path, err + ) + }); + let definition = manifest_decode(&definition).unwrap_or_else(|err| { + panic!( + "Failed to parse manifest definition from path {:?} - {:?}", + &coverage_path, err + ) + }); + return (code, definition); + } + }; + + // Build + let features = vec![ + "scrypto/log-error", + "scrypto/log-warn", + "scrypto/log-info", + "scrypto/log-debug", + "scrypto/log-trace", + ]; + + let status = Command::new("cargo") + .envs(env_vars) + .current_dir(package_dir.as_ref()) + .args([ + "build", + "--target", + "wasm32-unknown-unknown", + "--release", + "--features", + &features.join(","), + ]) + .status() + .unwrap_or_else(|error| { + panic!( + "Compiling \"{:?}\" failed with \"{:?}\"", + package_dir.as_ref(), + error + ) + }); + if !status.success() { + panic!("Failed to compile package: {:?}", package_dir.as_ref()); + } + // Extract schema let code = fs::read(&path).unwrap_or_else(|err| { panic!( diff --git a/scrypto/Cargo.toml b/scrypto/Cargo.toml index f1dd6581d73..860dea2df83 100644 --- a/scrypto/Cargo.toml +++ b/scrypto/Cargo.toml @@ -19,6 +19,7 @@ paste = { version = "1.0.13" } serde = { version = "1.0.144", default-features = false, optional = true } strum = { version = "0.24", default-features = false, features = ["derive"] } const-sha1 = { git = "https://github.com/radixdlt/const-sha1", default-features = false } # Chosen because of its small size and 0 transitive dependencies +minicov = { version = "0.3", optional = true } [features] # You should enable either `std` or `alloc` @@ -38,6 +39,9 @@ log-info = [] log-debug = [] log-trace = [] +# Feature to generate code coverage for WASM +coverage = ["minicov"] + [lib] doctest = false bench = false diff --git a/scrypto/src/lib.rs b/scrypto/src/lib.rs index cccd81d26ac..18d46305c13 100644 --- a/scrypto/src/lib.rs +++ b/scrypto/src/lib.rs @@ -35,6 +35,7 @@ pub use macros::*; // Re-export Scrypto derive. extern crate scrypto_derive; + pub use scrypto_derive::{blueprint, NonFungibleData}; // Re-export Radix Engine Interface modules. @@ -78,3 +79,11 @@ pub fn set_up_panic_hook() { crate::runtime::Runtime::panic(message); })); } + +#[cfg(all(feature = "coverage"))] +#[no_mangle] +pub unsafe extern "C" fn dump_coverage() -> types::Slice { + let mut coverage = vec![]; + minicov::capture_coverage(&mut coverage).unwrap(); + engine::wasm_api::forget_vec(coverage) +} diff --git a/simulator/Cargo.lock b/simulator/Cargo.lock index e1b54fd2efa..af36d63b91b 100644 --- a/simulator/Cargo.lock +++ b/simulator/Cargo.lock @@ -1609,6 +1609,7 @@ dependencies = [ "tempfile", "transaction", "utils", + "walkdir", "wasm-opt", ] diff --git a/simulator/Cargo.toml b/simulator/Cargo.toml index cf3224646a6..80f27d30cd6 100644 --- a/simulator/Cargo.toml +++ b/simulator/Cargo.toml @@ -35,6 +35,7 @@ proc-macro2 = { version = "1.0.38" } heck = "0.4.1" tempfile = "3.8.0" flume = { version = "0.11.0" } +walkdir = "2.3.3" [[bin]] name = "resim" diff --git a/simulator/src/resim/cmd_publish.rs b/simulator/src/resim/cmd_publish.rs index ae8ae923625..1185bfae69c 100644 --- a/simulator/src/resim/cmd_publish.rs +++ b/simulator/src/resim/cmd_publish.rs @@ -66,6 +66,7 @@ impl Publish { false, self.disable_wasm_opt, self.log_level.unwrap_or(Level::default()), + false, ) .map_err(Error::BuildError)? } else { diff --git a/simulator/src/scrypto/cmd_build.rs b/simulator/src/scrypto/cmd_build.rs index ac2c462e9bb..211cebf7304 100644 --- a/simulator/src/scrypto/cmd_build.rs +++ b/simulator/src/scrypto/cmd_build.rs @@ -34,6 +34,7 @@ impl Build { false, self.disable_wasm_opt, self.log_level.unwrap_or(Level::default()), + false, ) .map(|_| ()) .map_err(Error::BuildError) diff --git a/simulator/src/scrypto/cmd_coverage.rs b/simulator/src/scrypto/cmd_coverage.rs new file mode 100644 index 00000000000..6c6eb957c99 --- /dev/null +++ b/simulator/src/scrypto/cmd_coverage.rs @@ -0,0 +1,204 @@ +use clap::Parser; +use radix_engine_interface::types::Level; +use regex::Regex; +use std::env; +use std::env::current_dir; +use std::fs; +use std::io::Read; +use std::io::Write; +use std::path::PathBuf; +use std::process::Command; +use walkdir::WalkDir; + +use crate::scrypto::*; +use crate::utils::*; + +/// Run Scrypto tests and generate code coverage report +#[derive(Parser, Debug)] +pub struct Coverage { + /// The arguments to be passed to the test executable + arguments: Vec, + + /// The package directory + #[clap(long)] + path: Option, +} + +impl Coverage { + fn check_command_availability(command: String) -> Result<(), Error> { + if Command::new(&command).arg("--version").output().is_err() { + println!("Missing command: {}. Please install LLVM version matching rustc LLVM version, which is {}.", + command, command.split('-').last().unwrap_or("Unknown")); + println!("For more information, check https://apt.llvm.org/"); + return Err(Error::CoverageError(CoverageError::MissingLLVM)); + } + Ok(()) + } + + pub fn run(&self) -> Result<(), Error> { + let output = Command::new("rustc") + .arg("--version") + .arg("--verbose") + .output() + .expect("Failed to execute rustc command"); + + let output_str = String::from_utf8(output.stdout).expect("Failed to read rustc output"); + let is_nightly = output_str.contains("nightly"); + let llvm_major_version = Regex::new(r"LLVM version: (\d+)") + .unwrap() + .captures(&output_str) + .and_then(|cap| cap.get(1).map(|m| m.as_str())) + .expect("Failed to read LLVM version of rustc"); + + if !is_nightly { + println!("Coverage tool requries nightly version of rust toolchain"); + println!("You can install it by using the following commands:"); + println!("rustup default nightly"); + println!("rustup target add wasm32-unknown-unknown --toolchain=nightly"); + return Err(Error::CoverageError(CoverageError::IncorrectRustVersion)); + } + + // Verify that all llvm tools required to generate coverage report are installed + Self::check_command_availability(format!("clang-{}", llvm_major_version))?; + Self::check_command_availability(format!("llvm-cov-{}", llvm_major_version))?; + Self::check_command_availability(format!("llvm-profdata-{}", llvm_major_version))?; + + // Build package + let path = self.path.clone().unwrap_or(current_dir().unwrap()); + let (wasm_path, _) = build_package(&path, false, false, true, Level::Trace, true) + .map_err(Error::BuildError)?; + assert!(wasm_path.is_file()); + + // Remove wasm32-unknown-unknown/release/file.wasm from wasm_path + let mut coverage_path = wasm_path.clone(); + coverage_path.pop(); + coverage_path.pop(); + coverage_path.pop(); + assert!(coverage_path.ends_with("coverage")); + assert!(coverage_path.is_dir()); + + // Remove "data" directory from coverage directory if it exists, then create it + let data_path = coverage_path.join("data"); + if data_path.exists() { + fs::remove_dir_all(&data_path).unwrap(); + } + fs::create_dir_all(&data_path).unwrap(); + + // Set enviromental variable COVERAGE_DIRECTORY + env::set_var("COVERAGE_DIRECTORY", data_path.to_str().unwrap()); + + // Run tests + test_package(path, self.arguments.clone(), true) + .map(|_| ()) + .map_err(Error::TestError)?; + + // Merge profraw files into profdata file + let profraw_files: Vec = WalkDir::new(&data_path) + .into_iter() + .filter_map(|entry| { + let entry = entry.ok()?; + let path = entry.path(); + if path.extension()? == "profraw" { + Some(path.to_str()?.to_owned()) + } else { + None + } + }) + .collect(); + + if profraw_files.is_empty() { + println!("No .profraw files found in the coverage/data directory"); + return Err(Error::CoverageError(CoverageError::NoProfrawFiles)); + } + + let profdata_path = data_path.join("coverage.profdata"); + let output = Command::new(format!("llvm-profdata-{}", llvm_major_version)) + .args(&["merge", "-sparse"]) + .args(profraw_files) + .arg("-o") + .arg(profdata_path.to_str().unwrap()) + .output() + .expect("Failed to execute llvm-profdata command"); + if !output.status.success() { + eprintln!( + "llvm-profdata failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + return Err(Error::CoverageError(CoverageError::ProfdataMergeFailed)); + } + + // Generate object file from intermediate representation (.ll) file + let ir_path = wasm_path.with_extension("ll"); + let ir_path = ir_path + .parent() + .unwrap() + .join("deps") + .join(ir_path.file_name().unwrap()); + + let mut ir_contents = String::new(); + fs::File::open(&ir_path) + .expect(&format!("Failed to open IR file {ir_path:?}")) + .read_to_string(&mut ir_contents) + .expect("Failed to read IR file"); + + let modified_ir_contents = Regex::new(r"(?ms)^(define[^\n]*\n).*?^}\s*$") + .unwrap() + .replace_all(&ir_contents, "${1}start:\n unreachable\n}\n") + .to_string(); + + let new_ir_path = data_path.join(ir_path.file_name().unwrap()); + let mut new_ir_file = + fs::File::create(&new_ir_path).expect("Failed to create modified IR file"); + new_ir_file + .write_all(modified_ir_contents.as_bytes()) + .expect("Failed to write modified IR file"); + + // Generate Object File from IR File + let object_file_path = new_ir_path.with_extension("o"); + let output = Command::new(format!("clang-{}", llvm_major_version)) + .args(&[ + new_ir_path.to_str().unwrap(), + "-Wno-override-module", + "-c", + "-o", + ]) + .arg(object_file_path.to_str().unwrap()) + .output() + .expect("Failed to execute clang command"); + + if !output.status.success() { + eprintln!("clang failed: {}", String::from_utf8_lossy(&output.stderr)); + return Err(Error::CoverageError(CoverageError::ClangFailed)); + } + + // Generate Coverage Report + let coverage_report_path = coverage_path.join("report"); + if coverage_report_path.exists() { + fs::remove_dir_all(&coverage_report_path).unwrap(); + } + + let output = Command::new(format!("llvm-cov-{}", llvm_major_version)) + .args(&["show", "--instr-profile=coverage/data/coverage.profdata"]) + .arg(object_file_path.to_str().unwrap()) + .args(&[ + "--show-instantiations=false", + "--format=html", + "--output-dir", + ]) + .arg(coverage_report_path.to_str().unwrap()) + .output() + .expect("Failed to execute llvm-cov command"); + + if !output.status.success() { + eprintln!( + "llvm-cov failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + return Err(Error::CoverageError(CoverageError::LlvmCovFailed)); + } + + println!("Coverage report was succesfully generated, it is available in {coverage_report_path:?} directory."); + + Ok(()) + } +} diff --git a/simulator/src/scrypto/cmd_test.rs b/simulator/src/scrypto/cmd_test.rs index 627e19db4b9..f943391e205 100644 --- a/simulator/src/scrypto/cmd_test.rs +++ b/simulator/src/scrypto/cmd_test.rs @@ -21,6 +21,7 @@ impl Test { test_package( self.path.clone().unwrap_or(current_dir().unwrap()), self.arguments.clone(), + false, ) .map(|_| ()) .map_err(Error::TestError) diff --git a/simulator/src/scrypto/error.rs b/simulator/src/scrypto/error.rs index 21caf795c47..499bf52503b 100644 --- a/simulator/src/scrypto/error.rs +++ b/simulator/src/scrypto/error.rs @@ -13,4 +13,6 @@ pub enum Error { FormatError(FormatError), PackageAlreadyExists, + + CoverageError(CoverageError), } diff --git a/simulator/src/scrypto/mod.rs b/simulator/src/scrypto/mod.rs index 1019ec12276..8d9588e8c36 100644 --- a/simulator/src/scrypto/mod.rs +++ b/simulator/src/scrypto/mod.rs @@ -1,10 +1,12 @@ mod cmd_build; +mod cmd_coverage; mod cmd_fmt; mod cmd_new_package; mod cmd_test; mod error; pub use cmd_build::*; +pub use cmd_coverage::*; pub use cmd_fmt::*; pub use cmd_new_package::*; pub use cmd_test::*; @@ -23,6 +25,7 @@ pub struct ScryptoCli { #[derive(Subcommand, Debug)] pub enum Command { Build(Build), + Coverage(Coverage), Fmt(Fmt), NewPackage(NewPackage), Test(Test), @@ -33,6 +36,7 @@ pub fn run() -> Result<(), Error> { match cli.command { Command::Build(cmd) => cmd.run(), + Command::Coverage(cmd) => cmd.run(), Command::Fmt(cmd) => cmd.run(), Command::NewPackage(cmd) => cmd.run(), Command::Test(cmd) => cmd.run(), diff --git a/simulator/src/scrypto_bindgen/mod.rs b/simulator/src/scrypto_bindgen/mod.rs index 7eb38601ab1..1744d8020f7 100644 --- a/simulator/src/scrypto_bindgen/mod.rs +++ b/simulator/src/scrypto_bindgen/mod.rs @@ -117,7 +117,10 @@ where S: SubstateDatabase, { fn lookup_schema(&self, schema_hash: &SchemaHash) -> Option { - self.1.get_schema(self.0.as_node_id(), schema_hash).ok().map(|x| x.as_ref().clone()) + self.1 + .get_schema(self.0.as_node_id(), schema_hash) + .ok() + .map(|x| x.as_ref().clone()) } fn resolve_type_kind( diff --git a/simulator/src/utils/cargo.rs b/simulator/src/utils/cargo.rs index 1a45ac89697..96fcdaf9558 100644 --- a/simulator/src/utils/cargo.rs +++ b/simulator/src/utils/cargo.rs @@ -1,3 +1,4 @@ +use std::env; use std::ffi::OsStr; use std::fs; use std::io; @@ -60,6 +61,7 @@ fn run_cargo_build( trace: bool, no_schema: bool, log_level: Level, + coverage: bool, ) -> Result<(), BuildError> { let mut features = String::new(); if trace { @@ -83,6 +85,15 @@ fn run_cargo_build( if Level::Trace <= log_level { features.push_str(",scrypto/log-trace"); } + if coverage { + features.push_str(",scrypto/coverage"); + } + + let rustflags = if coverage { + "-Clto=off\x1f-Cinstrument-coverage\x1f-Zno-profiler-runtime\x1f--emit=llvm-ir".to_owned() + } else { + env::var("CARGO_ENCODED_RUSTFLAGS").unwrap_or_default() + }; let status = Command::new("cargo") .arg("build") @@ -98,6 +109,7 @@ fn run_cargo_build( } else { vec!["--features", &features[1..]] }) + .env("CARGO_ENCODED_RUSTFLAGS", rustflags) .status() .map_err(BuildError::IOError)?; if status.success() { @@ -142,6 +154,7 @@ pub fn build_package>( force_local_target: bool, disable_wasm_opt: bool, log_level: Level, + coverage: bool, ) -> Result<(PathBuf, PathBuf), BuildError> { let base_path = base_path.as_ref().to_owned(); @@ -154,21 +167,26 @@ pub fn build_package>( // Use the scrypto directory as a target, even if the scrypto crate is part of a workspace // This allows us to find where the WASM and SCHEMA ends up deterministically. - let target_path = if force_local_target { + let mut target_path = if force_local_target { let mut target_path = base_path.clone(); target_path.push("target"); target_path } else { PathBuf::from_str(&get_default_target_directory(&manifest_path)?).unwrap() - // Infallible }; + // If coverage is enabled we change target directory to coverage directory + if coverage { + target_path.pop(); + target_path.push("coverage"); + } + let mut out_path = target_path.clone(); out_path.push("wasm32-unknown-unknown"); out_path.push("release"); // Build with SCHEMA - run_cargo_build(&manifest_path, &target_path, trace, false, log_level)?; + run_cargo_build(&manifest_path, &target_path, trace, false, log_level, false)?; // Find the binary paths let manifest = Manifest::from_path(&manifest_path) @@ -199,7 +217,14 @@ pub fn build_package>( .map_err(|err| BuildError::IOErrorAtPath(err, definition_path.clone()))?; // Build without SCHEMA - run_cargo_build(&manifest_path, &target_path, trace, true, log_level)?; + run_cargo_build( + &manifest_path, + &target_path, + trace, + true, + log_level, + coverage, + )?; // Optimizes the built wasm using Binaryen's wasm-opt tool. The code that follows is equivalent // to running the following commands in the CLI: @@ -217,21 +242,30 @@ pub fn build_package>( } /// Runs tests within a package. -pub fn test_package, I, S>(path: P, args: I) -> Result<(), TestError> +pub fn test_package, I, S>(path: P, args: I, coverage: bool) -> Result<(), TestError> where I: IntoIterator, S: AsRef, { - build_package(&path, false, false, false, Level::Trace).map_err(TestError::BuildError)?; + if !coverage { + build_package(&path, false, false, false, Level::Trace, false) + .map_err(TestError::BuildError)?; + } let mut cargo = path.as_ref().to_owned(); cargo.push("Cargo.toml"); if cargo.exists() { + let features = if coverage { + vec!["--features", "scrypto-unit/coverage"] + } else { + vec![] + }; let status = Command::new("cargo") .arg("test") .arg("--release") .arg("--manifest-path") .arg(cargo.to_str().unwrap()) + .args(features) .arg("--") .args(args) .status() diff --git a/simulator/src/utils/coverage.rs b/simulator/src/utils/coverage.rs new file mode 100644 index 00000000000..9829446a0b6 --- /dev/null +++ b/simulator/src/utils/coverage.rs @@ -0,0 +1,9 @@ +#[derive(Debug)] +pub enum CoverageError { + IncorrectRustVersion, + MissingLLVM, + NoProfrawFiles, + ProfdataMergeFailed, + ClangFailed, + LlvmCovFailed, +} diff --git a/simulator/src/utils/mod.rs b/simulator/src/utils/mod.rs index 9acac82dca8..88ba5e34d9d 100644 --- a/simulator/src/utils/mod.rs +++ b/simulator/src/utils/mod.rs @@ -1,11 +1,13 @@ mod cargo; mod common_instructions; +mod coverage; mod display; mod iter; mod resource_specifier; pub use cargo::*; pub use common_instructions::*; +pub use coverage::*; pub use display::list_item_prefix; pub use iter::{IdentifyLast, Iter}; pub use resource_specifier::*; From 0b1cfa761a6d020e6853ef660e31b5292d7e119b Mon Sep 17 00:00:00 2001 From: bbarwik Date: Fri, 10 Nov 2023 13:43:25 +0100 Subject: [PATCH 2/3] Improved scrypto coverage command --- radix-engine/src/vm/wasm/wasmer.rs | 21 +++--- scrypto-unit/src/test_runner.rs | 4 +- simulator/src/scrypto/cmd_coverage.rs | 93 ++++++++++++++++++++------- simulator/src/utils/coverage.rs | 1 + 4 files changed, 85 insertions(+), 34 deletions(-) diff --git a/radix-engine/src/vm/wasm/wasmer.rs b/radix-engine/src/vm/wasm/wasmer.rs index faf5e62c704..17d65db8d78 100644 --- a/radix-engine/src/vm/wasm/wasmer.rs +++ b/radix-engine/src/vm/wasm/wasmer.rs @@ -842,12 +842,13 @@ impl WasmInstance for WasmerInstance { let result = match return_data { Ok(data) => { if let Some(v) = data.as_ref().get(0).and_then(|x| x.i64()) { - read_slice(&self.instance, Slice::transmute_i64(v)).map_err(InvokeError::SelfError) + read_slice(&self.instance, Slice::transmute_i64(v)) + .map_err(InvokeError::SelfError) } else { Err(InvokeError::SelfError(WasmRuntimeError::InvalidWasmPointer)) - } + } } - Err(err) => Err(err) + Err(err) => Err(err), }; #[cfg(feature = "coverage")] @@ -857,12 +858,14 @@ impl WasmInstance for WasmerInstance { String::from_utf8(runtime.buffer_consume(blueprint_buffer.id()).unwrap()) .unwrap(); - let mut ret = dump_coverage.call(&[]).unwrap().as_ref().get(0).and_then(|x| x.i64()).unwrap(); - let coverage_data = read_slice( - &self.instance, - Slice::transmute_i64(ret), - ) - .unwrap(); + let mut ret = dump_coverage + .call(&[]) + .unwrap() + .as_ref() + .get(0) + .and_then(|x| x.i64()) + .unwrap(); + let coverage_data = read_slice(&self.instance, Slice::transmute_i64(ret)).unwrap(); save_coverage_data(&blueprint_name, &coverage_data); } } diff --git a/scrypto-unit/src/test_runner.rs b/scrypto-unit/src/test_runner.rs index 36b9960ede8..612c55a27e7 100644 --- a/scrypto-unit/src/test_runner.rs +++ b/scrypto-unit/src/test_runner.rs @@ -125,13 +125,13 @@ impl Compile { coverage_path.set_extension("rpd"); let definition = fs::read(&coverage_path).unwrap_or_else(|err| { panic!( - "Failed to read manifest definition from path {:?} - {:?}", + "Failed to read package definition from path {:?} - {:?}", &coverage_path, err ) }); let definition = manifest_decode(&definition).unwrap_or_else(|err| { panic!( - "Failed to parse manifest definition from path {:?} - {:?}", + "Failed to parse package definition from path {:?} - {:?}", &coverage_path, err ) }); diff --git a/simulator/src/scrypto/cmd_coverage.rs b/simulator/src/scrypto/cmd_coverage.rs index 6c6eb957c99..dd778b94b23 100644 --- a/simulator/src/scrypto/cmd_coverage.rs +++ b/simulator/src/scrypto/cmd_coverage.rs @@ -25,20 +25,34 @@ pub struct Coverage { } impl Coverage { - fn check_command_availability(command: String) -> Result<(), Error> { - if Command::new(&command).arg("--version").output().is_err() { - println!("Missing command: {}. Please install LLVM version matching rustc LLVM version, which is {}.", - command, command.split('-').last().unwrap_or("Unknown")); - println!("For more information, check https://apt.llvm.org/"); - return Err(Error::CoverageError(CoverageError::MissingLLVM)); + fn check_wasm_target(nightly: bool) -> Result<(), Error> { + let output = Command::new("rustup") + .args(&["target", "list", "--installed"]) + .output() + .expect("Failed to execute rustup target list command"); + + let output_str = String::from_utf8(output.stdout).unwrap(); + let is_wasm_target_installed = output_str.contains("wasm32-unknown-unknown"); + + if !is_wasm_target_installed { + eprintln!( + "The {}wasm32-unknown-unknown target is not installed.", + if nightly { "nightly" } else { "" } + ); + eprintln!("You can install it by using the following command:"); + eprintln!( + "rustup target add wasm32-unknown-unknown{}", + if nightly { " --toolchain=nightly" } else { "" } + ); + Err(Error::CoverageError(CoverageError::MissingWasm32Target)) + } else { + Ok(()) } - Ok(()) } - pub fn run(&self) -> Result<(), Error> { + fn check_rustc_version() -> (bool, String) { let output = Command::new("rustc") - .arg("--version") - .arg("--verbose") + .args(&["--version", "--verbose"]) .output() .expect("Failed to execute rustc command"); @@ -48,14 +62,41 @@ impl Coverage { .unwrap() .captures(&output_str) .and_then(|cap| cap.get(1).map(|m| m.as_str())) - .expect("Failed to read LLVM version of rustc"); + .map(String::from) + .unwrap(); + (is_nightly, llvm_major_version) + } + + fn check_command_availability(command: String) -> Result<(), Error> { + if Command::new(&command).arg("--version").output().is_err() { + eprintln!("Missing command: {}. Please install LLVM version matching rustc LLVM version, which is {}.", + command, command.split('-').last().unwrap_or("Unknown")); + eprintln!("For more information, check https://apt.llvm.org/"); + Err(Error::CoverageError(CoverageError::MissingLLVM)) + } else { + Ok(()) + } + } + + pub fn run(&self) -> Result<(), Error> { + // Verify rust version and wasm target + Self::check_wasm_target(false)?; + + let (mut is_nightly, mut llvm_major_version) = Self::check_rustc_version(); + let mut unset_rustup_toolchain = false; if !is_nightly { - println!("Coverage tool requries nightly version of rust toolchain"); - println!("You can install it by using the following commands:"); - println!("rustup default nightly"); - println!("rustup target add wasm32-unknown-unknown --toolchain=nightly"); - return Err(Error::CoverageError(CoverageError::IncorrectRustVersion)); + // Try to use nightly toolchain automatically + env::set_var("RUSTUP_TOOLCHAIN", "nightly"); + (is_nightly, llvm_major_version) = Self::check_rustc_version(); + if !is_nightly { + eprintln!("Coverage tool requries nightly version of rust toolchain"); + eprintln!("You can install it by using the following commands:"); + eprintln!("rustup target add wasm32-unknown-unknown --toolchain=nightly"); + return Err(Error::CoverageError(CoverageError::IncorrectRustVersion)); + } + Self::check_wasm_target(true)?; + unset_rustup_toolchain = true; } // Verify that all llvm tools required to generate coverage report are installed @@ -69,6 +110,10 @@ impl Coverage { .map_err(Error::BuildError)?; assert!(wasm_path.is_file()); + if unset_rustup_toolchain { + env::remove_var("RUSTUP_TOOLCHAIN"); + } + // Remove wasm32-unknown-unknown/release/file.wasm from wasm_path let mut coverage_path = wasm_path.clone(); coverage_path.pop(); @@ -107,7 +152,7 @@ impl Coverage { .collect(); if profraw_files.is_empty() { - println!("No .profraw files found in the coverage/data directory"); + eprintln!("No .profraw files found in the coverage/data directory"); return Err(Error::CoverageError(CoverageError::NoProfrawFiles)); } @@ -115,8 +160,7 @@ impl Coverage { let output = Command::new(format!("llvm-profdata-{}", llvm_major_version)) .args(&["merge", "-sparse"]) .args(profraw_files) - .arg("-o") - .arg(profdata_path.to_str().unwrap()) + .args(&["-o", profdata_path.to_str().unwrap()]) .output() .expect("Failed to execute llvm-profdata command"); if !output.status.success() { @@ -141,6 +185,7 @@ impl Coverage { .read_to_string(&mut ir_contents) .expect("Failed to read IR file"); + // Modify IR file according to https://github.com/hknio/code-coverage-for-webassembly let modified_ir_contents = Regex::new(r"(?ms)^(define[^\n]*\n).*?^}\s*$") .unwrap() .replace_all(&ir_contents, "${1}start:\n unreachable\n}\n") @@ -161,8 +206,8 @@ impl Coverage { "-Wno-override-module", "-c", "-o", + object_file_path.to_str().unwrap(), ]) - .arg(object_file_path.to_str().unwrap()) .output() .expect("Failed to execute clang command"); @@ -178,14 +223,16 @@ impl Coverage { } let output = Command::new(format!("llvm-cov-{}", llvm_major_version)) - .args(&["show", "--instr-profile=coverage/data/coverage.profdata"]) - .arg(object_file_path.to_str().unwrap()) .args(&[ + "show", + "--instr-profile", + profdata_path.to_str().unwrap(), + object_file_path.to_str().unwrap(), "--show-instantiations=false", "--format=html", "--output-dir", + coverage_report_path.to_str().unwrap(), ]) - .arg(coverage_report_path.to_str().unwrap()) .output() .expect("Failed to execute llvm-cov command"); diff --git a/simulator/src/utils/coverage.rs b/simulator/src/utils/coverage.rs index 9829446a0b6..ff792d2f14e 100644 --- a/simulator/src/utils/coverage.rs +++ b/simulator/src/utils/coverage.rs @@ -1,5 +1,6 @@ #[derive(Debug)] pub enum CoverageError { + MissingWasm32Target, IncorrectRustVersion, MissingLLVM, NoProfrawFiles, From c9e251d8e36368def9974fd0a25143c114efbf1e Mon Sep 17 00:00:00 2001 From: bbarwik Date: Fri, 10 Nov 2023 14:00:26 +0100 Subject: [PATCH 3/3] Added test for scrypto coverage command --- simulator/tests/scrypto_coverage.sh | 30 +++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 simulator/tests/scrypto_coverage.sh diff --git a/simulator/tests/scrypto_coverage.sh b/simulator/tests/scrypto_coverage.sh new file mode 100755 index 00000000000..fae21bfcf4a --- /dev/null +++ b/simulator/tests/scrypto_coverage.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# This script requires rust nightly and wasm32-unknown-unknown target. +# Additionally, LLVM needs to be installed and its version should match the major version of your rust nightly compiler. + +set -x +set -e + +cd "$(dirname "$0")/.." + +scrypto="cargo run --bin scrypto $@ --" +test_pkg="./target/temp/hello-world" + +# Create package +rm -fr $test_pkg +$scrypto new-package hello-world --path $test_pkg --local + +# Generate coverage report +$scrypto coverage --path $test_pkg + +# Check if coverage report was generated +if [ -f "$test_pkg/coverage/report/index.html" ]; then + echo "Coverage report generated successfully." +else + echo "Error: Coverage report not found." + exit 1 +fi + +# Clean up +rm -fr $test_pkg \ No newline at end of file