diff --git a/Cargo.lock b/Cargo.lock index e93a6d62..45f2e147 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", @@ -254,6 +254,7 @@ version = "3.0.0" dependencies = [ "anyhow", "async-compression", + "async-trait", "base64", "clap", "console", diff --git a/Cargo.toml b/Cargo.toml index a6d310d0..f032e6d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ serde_yaml = "0.9.34" sysinfo = { version = "0.30.12", features = ["serde"] } indicatif = "0.17.8" console = "0.15.8" +async-trait = "0.1.82" [dev-dependencies] temp-env = { version = "0.3.6", features = ["async_closure"] } diff --git a/src/run/check_system.rs b/src/run/check_system.rs index 360fb2a1..ed15ee0b 100644 --- a/src/run/check_system.rs +++ b/src/run/check_system.rs @@ -3,7 +3,7 @@ use std::process::Command; use lazy_static::lazy_static; use serde::{Deserialize, Serialize}; use std::collections::HashSet; -use sysinfo::System; +use sysinfo::{CpuRefreshKind, MemoryRefreshKind, RefreshKind, System}; use crate::prelude::*; @@ -27,6 +27,11 @@ pub struct SystemInfo { pub arch: String, pub host: String, pub user: String, + pub cpu_brand: String, + pub cpu_name: String, + pub cpu_vendor_id: String, + pub cpu_cores: usize, + pub total_memory_gb: u64, } #[cfg(test)] @@ -38,6 +43,11 @@ impl SystemInfo { arch: "x86_64".to_string(), host: "host".to_string(), user: "user".to_string(), + cpu_brand: "Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz".to_string(), + cpu_name: "cpu0".to_string(), + cpu_vendor_id: "GenuineIntel".to_string(), + cpu_cores: 2, + total_memory_gb: 8, } } } @@ -50,12 +60,37 @@ impl SystemInfo { let user = get_user()?; let host = System::host_name().ok_or(anyhow!("Failed to get host name"))?; + let s = System::new_with_specifics( + RefreshKind::new() + .with_cpu(CpuRefreshKind::everything()) + .with_memory(MemoryRefreshKind::everything()), + ); + let cpu_cores = s + .physical_core_count() + .ok_or(anyhow!("Failed to get CPU core count"))?; + let total_memory_gb = s.total_memory().div_ceil(1024_u64.pow(3)); + + // take the first CPU to get the brand, name and vendor id + let cpu = s + .cpus() + .iter() + .next() + .ok_or(anyhow!("Failed to get CPU info"))?; + let cpu_brand = cpu.brand().to_string(); + let cpu_name = cpu.name().to_string(); + let cpu_vendor_id = cpu.vendor_id().to_string(); + Ok(SystemInfo { os, os_version, arch, host, user, + cpu_brand, + cpu_name, + cpu_vendor_id, + cpu_cores, + total_memory_gb, }) } } diff --git a/src/run/ci_provider/provider.rs b/src/run/ci_provider/provider.rs index fb581b4b..7d009971 100644 --- a/src/run/ci_provider/provider.rs +++ b/src/run/ci_provider/provider.rs @@ -4,6 +4,7 @@ use simplelog::SharedLogger; use crate::prelude::*; use crate::run::check_system::SystemInfo; use crate::run::config::Config; +use crate::run::runner::ExecutorName; use crate::run::uploader::{Runner, UploadMetadata}; use super::interfaces::ProviderMetadata; @@ -77,13 +78,14 @@ pub trait CIProvider { config: &Config, system_info: &SystemInfo, archive_hash: &str, + executor_name: ExecutorName, ) -> Result { let provider_metadata = self.get_provider_metadata()?; let commit_hash = get_commit_hash(&provider_metadata.repository_root_path)?; Ok(UploadMetadata { - version: Some(3), + version: Some(4), tokenless: config.token.is_none(), provider_metadata, profile_md5: archive_hash.into(), @@ -92,6 +94,7 @@ pub trait CIProvider { name: "codspeed-runner".into(), version: crate::VERSION.into(), instruments: config.instruments.get_active_instrument_names(), + executor: executor_name, system_info: system_info.clone(), }, platform: self.get_provider_slug().into(), diff --git a/src/run/instruments/mod.rs b/src/run/instruments/mod.rs index 8e2e79bb..752a41e7 100644 --- a/src/run/instruments/mod.rs +++ b/src/run/instruments/mod.rs @@ -19,7 +19,7 @@ pub struct Instruments { } #[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash)] -pub enum InstrumentNames { +pub enum InstrumentName { MongoDB, } @@ -28,11 +28,11 @@ impl Instruments { self.mongodb.is_some() } - pub fn get_active_instrument_names(&self) -> Vec { + pub fn get_active_instrument_names(&self) -> Vec { let mut names = vec![]; if self.is_mongodb_enabled() { - names.push(InstrumentNames::MongoDB); + names.push(InstrumentName::MongoDB); } names @@ -42,16 +42,16 @@ impl Instruments { impl TryFrom<&RunArgs> for Instruments { type Error = Error; fn try_from(args: &RunArgs) -> Result { - let mut validated_instrument_names: HashSet = HashSet::new(); + let mut validated_instrument_names: HashSet = HashSet::new(); for instrument_name in &args.instruments { match instrument_name.as_str() { - "mongodb" => validated_instrument_names.insert(InstrumentNames::MongoDB), + "mongodb" => validated_instrument_names.insert(InstrumentName::MongoDB), _ => bail!("Invalid instrument name: {}", instrument_name), }; } - let mongodb = if validated_instrument_names.contains(&InstrumentNames::MongoDB) { + let mongodb = if validated_instrument_names.contains(&InstrumentName::MongoDB) { Some(MongoDBConfig { uri_env_name: args.mongo_uri_env_name.clone(), }) diff --git a/src/run/instruments/mongo_tracer.rs b/src/run/instruments/mongo_tracer.rs index 9a6d4dc7..7783a1d8 100644 --- a/src/run/instruments/mongo_tracer.rs +++ b/src/run/instruments/mongo_tracer.rs @@ -49,7 +49,7 @@ fn dump_tracer_log(mut stream: impl Read) -> Result<()> { } impl MongoTracer { - pub fn try_from(profile_folder: &PathBuf, mongodb_config: &MongoDBConfig) -> Result { + pub fn try_from(profile_folder: &Path, mongodb_config: &MongoDBConfig) -> Result { let user_input = match &mongodb_config.uri_env_name { Some(uri_env_name) => { debug!( diff --git a/src/run/mod.rs b/src/run/mod.rs index 7324667b..c0931e8d 100644 --- a/src/run/mod.rs +++ b/src/run/mod.rs @@ -5,6 +5,8 @@ use crate::run::{config::Config, logger::Logger}; use crate::VERSION; use check_system::SystemInfo; use clap::Args; +use instruments::mongo_tracer::MongoTracer; +use runner::get_run_data; mod check_system; pub mod ci_provider; @@ -114,12 +116,45 @@ pub async fn run(args: RunArgs, api_client: &CodSpeedAPIClient) -> Result<()> { let system_info = SystemInfo::new()?; check_system::check_system(&system_info)?; - let run_data = runner::run(&config, &system_info).await?; + let executor = runner::get_executor()?; + + let run_data = get_run_data()?; + + if !config.skip_setup { + start_group!("Preparing the environment"); + executor.setup(&config, &system_info, &run_data).await?; + end_group!(); + } + + start_opened_group!("Running the benchmarks"); + + // TODO: refactor and move directly in the Instruments struct as a `start` method + let mongo_tracer = if let Some(mongodb_config) = &config.instruments.mongodb { + let mut mongo_tracer = MongoTracer::try_from(&run_data.profile_folder, mongodb_config)?; + mongo_tracer.start().await?; + Some(mongo_tracer) + } else { + None + }; + + executor + .run(&config, &system_info, &run_data, &mongo_tracer) + .await?; + + // TODO: refactor and move directly in the Instruments struct as a `stop` method + if let Some(mut mongo_tracer) = mongo_tracer { + mongo_tracer.stop().await?; + } + + executor.teardown(&config, &system_info, &run_data).await?; + + end_group!(); if !config.skip_upload { start_group!("Uploading performance data"); logger.persist_log_to_profile_folder(&run_data)?; - let upload_result = uploader::upload(&config, &system_info, &provider, &run_data).await?; + let upload_result = + uploader::upload(&config, &system_info, &provider, &run_data, executor.name()).await?; end_group!(); if provider.get_provider_slug() == "local" { diff --git a/src/run/runner/executor.rs b/src/run/runner/executor.rs new file mode 100644 index 00000000..9332596f --- /dev/null +++ b/src/run/runner/executor.rs @@ -0,0 +1,35 @@ +use async_trait::async_trait; + +use super::interfaces::{ExecutorName, RunData}; +use crate::prelude::*; +use crate::run::instruments::mongo_tracer::MongoTracer; +use crate::run::{check_system::SystemInfo, config::Config}; + +#[async_trait(?Send)] +pub trait Executor { + fn name(&self) -> ExecutorName; + + async fn setup( + &self, + config: &Config, + system_info: &SystemInfo, + run_data: &RunData, + ) -> Result<()>; + + /// Runs the executor + async fn run( + &self, + config: &Config, + system_info: &SystemInfo, + run_data: &RunData, + // TODO: use Instruments instead of directly passing the mongodb tracer + mongo_tracer: &Option, + ) -> Result<()>; + + async fn teardown( + &self, + config: &Config, + system_info: &SystemInfo, + run_data: &RunData, + ) -> Result<()>; +} diff --git a/src/run/runner/helpers/env.rs b/src/run/runner/helpers/env.rs new file mode 100644 index 00000000..f6b3fd9a --- /dev/null +++ b/src/run/runner/helpers/env.rs @@ -0,0 +1,14 @@ +use std::{collections::HashMap, env::consts::ARCH}; + +use lazy_static::lazy_static; + +lazy_static! { + pub static ref BASE_INJECTED_ENV: HashMap<&'static str, String> = { + HashMap::from([ + ("PYTHONMALLOC", "malloc".into()), + ("PYTHONHASHSEED", "0".into()), + ("ARCH", ARCH.into()), + ("CODSPEED_ENV", "runner".into()), + ]) + }; +} diff --git a/src/run/runner/helpers/get_bench_command.rs b/src/run/runner/helpers/get_bench_command.rs new file mode 100644 index 00000000..18732f09 --- /dev/null +++ b/src/run/runner/helpers/get_bench_command.rs @@ -0,0 +1,58 @@ +use crate::prelude::*; +use crate::run::config::Config; + +pub fn get_bench_command(config: &Config) -> Result { + let bench_command = &config.command.trim(); + + if bench_command.is_empty() { + bail!("The bench command is empty"); + } + + Ok(bench_command + // Fixes a compatibility issue with cargo 1.66+ running directly under valgrind <3.20 + .replace("cargo codspeed", "cargo-codspeed")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_bench_command_empty() { + let config = Config::test(); + assert!(get_bench_command(&config).is_err()); + assert_eq!( + get_bench_command(&config).unwrap_err().to_string(), + "The bench command is empty" + ); + } + + #[test] + fn test_get_bench_command_cargo() { + let config = Config { + command: "cargo codspeed bench".into(), + ..Config::test() + }; + assert_eq!(get_bench_command(&config).unwrap(), "cargo-codspeed bench"); + } + + #[test] + fn test_get_bench_command_multiline() { + let config = Config { + // TODO: use indoc! macro + command: r#" +cargo codspeed bench --features "foo bar" +pnpm vitest bench "my-app" +pytest tests/ --codspeed +"# + .into(), + ..Config::test() + }; + assert_eq!( + get_bench_command(&config).unwrap(), + r#"cargo-codspeed bench --features "foo bar" +pnpm vitest bench "my-app" +pytest tests/ --codspeed"# + ); + } +} diff --git a/src/run/runner/helpers/mod.rs b/src/run/runner/helpers/mod.rs index 41cd2b5f..87c246d9 100644 --- a/src/run/runner/helpers/mod.rs +++ b/src/run/runner/helpers/mod.rs @@ -1,5 +1,4 @@ -pub mod download_file; -pub mod ignored_objects_path; -pub mod introspected_node; -pub mod perf_maps; +pub mod env; +pub mod get_bench_command; pub mod profile_folder; +pub mod run_command_with_log_pipe; diff --git a/src/run/runner/helpers/run_command_with_log_pipe.rs b/src/run/runner/helpers/run_command_with_log_pipe.rs new file mode 100644 index 00000000..a6c2199c --- /dev/null +++ b/src/run/runner/helpers/run_command_with_log_pipe.rs @@ -0,0 +1,52 @@ +use crate::local_logger::suspend_progress_bar; +use crate::prelude::*; +use std::io::{Read, Write}; +use std::process::Command; +use std::process::ExitStatus; +use std::thread; + +pub fn run_command_with_log_pipe(mut cmd: Command, target: &str) -> Result { + fn log_tee( + mut reader: impl Read, + mut writer: impl Write, + log_prefix: Option<&str>, + target: &str, + ) -> Result<()> { + let prefix = log_prefix.unwrap_or(""); + let mut buffer = [0; 1024]; + loop { + let bytes_read = reader.read(&mut buffer)?; + if bytes_read == 0 { + break; + } + suspend_progress_bar(|| { + writer.write_all(&buffer[..bytes_read]).unwrap(); + trace!( + target: target, + "{}{}", + prefix, + String::from_utf8_lossy(&buffer[..bytes_read]) + ); + }); + } + Ok(()) + } + + let mut process = cmd + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("failed to spawn the process")?; + let stdout = process.stdout.take().expect("unable to get stdout"); + let stderr = process.stderr.take().expect("unable to get stderr"); + let target_clone = target.to_string(); + thread::spawn(move || { + log_tee(stdout, std::io::stdout(), None, &target_clone).unwrap(); + }); + + let target_clone = target.to_string(); + thread::spawn(move || { + log_tee(stderr, std::io::stderr(), Some("[stderr]"), &target_clone).unwrap(); + }); + process.wait().context("failed to wait for the process") +} diff --git a/src/run/runner/interfaces.rs b/src/run/runner/interfaces.rs new file mode 100644 index 00000000..66fbc6db --- /dev/null +++ b/src/run/runner/interfaces.rs @@ -0,0 +1,23 @@ +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +pub struct RunData { + pub profile_folder: PathBuf, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash)] +#[serde(rename_all = "lowercase")] +pub enum ExecutorName { + Valgrind, + WallTime, +} + +#[allow(clippy::to_string_trait_impl)] +impl ToString for ExecutorName { + fn to_string(&self) -> String { + match self { + ExecutorName::Valgrind => "valgrind".to_string(), + ExecutorName::WallTime => "walltime".to_string(), + } + } +} diff --git a/src/run/runner/mod.rs b/src/run/runner/mod.rs index 5ae8c8c4..61b03c9a 100644 --- a/src/run/runner/mod.rs +++ b/src/run/runner/mod.rs @@ -1,8 +1,37 @@ +use std::env; + +use crate::prelude::*; + +mod executor; mod helpers; -mod run; -mod setup; +mod interfaces; mod valgrind; +mod wall_time; + +use anyhow::bail; +use executor::Executor; +use helpers::profile_folder::create_profile_folder; +pub use interfaces::{ExecutorName, RunData}; +use valgrind::executor::{ValgrindExecutor, INSTRUMENTATION_RUNNER_MODE}; +use wall_time::executor::{WallTimeExecutor, WALL_TIME_RUNNER_MODE}; -pub use self::run::RunData; -pub use run::run; pub use valgrind::VALGRIND_EXECUTION_TARGET; + +pub fn get_executor() -> Result> { + if let Ok(runner_mode) = env::var("CODSPEED_RUNNER_MODE") { + debug!("CODSPEED_RUNNER_MODE is set to {}", runner_mode); + match runner_mode.as_str() { + INSTRUMENTATION_RUNNER_MODE => Ok(Box::new(ValgrindExecutor)), + WALL_TIME_RUNNER_MODE => Ok(Box::new(WallTimeExecutor)), + _ => bail!("Unknown codspeed runner mode"), + } + } else { + debug!("CODSPEED_RUNNER_MODE is not set, using valgrind"); + Ok(Box::new(ValgrindExecutor)) + } +} + +pub fn get_run_data() -> Result { + let profile_folder = create_profile_folder()?; + Ok(RunData { profile_folder }) +} diff --git a/src/run/runner/run.rs b/src/run/runner/run.rs deleted file mode 100644 index 46c78896..00000000 --- a/src/run/runner/run.rs +++ /dev/null @@ -1,41 +0,0 @@ -use crate::prelude::*; -use crate::run::{ - check_system::SystemInfo, config::Config, instruments::mongo_tracer::MongoTracer, -}; - -use std::path::PathBuf; - -use super::{ - helpers::{perf_maps::harvest_perf_maps, profile_folder::create_profile_folder}, - setup::setup, - valgrind, -}; - -pub struct RunData { - pub profile_folder: PathBuf, -} - -pub async fn run(config: &Config, system_info: &SystemInfo) -> Result { - if !config.skip_setup { - start_group!("Preparing the environment"); - setup(system_info, config).await?; - end_group!(); - } - //TODO: add valgrind version check - start_opened_group!("Running the benchmarks"); - let profile_folder = create_profile_folder()?; - let mongo_tracer = if let Some(mongodb_config) = &config.instruments.mongodb { - let mut mongo_tracer = MongoTracer::try_from(&profile_folder, mongodb_config)?; - mongo_tracer.start().await?; - Some(mongo_tracer) - } else { - None - }; - valgrind::measure(config, &profile_folder, &mongo_tracer)?; - harvest_perf_maps(&profile_folder)?; - if let Some(mut mongo_tracer) = mongo_tracer { - mongo_tracer.stop().await?; - } - end_group!(); - Ok(RunData { profile_folder }) -} diff --git a/src/run/runner/valgrind.rs b/src/run/runner/valgrind.rs deleted file mode 100644 index 3c6d7ea9..00000000 --- a/src/run/runner/valgrind.rs +++ /dev/null @@ -1,204 +0,0 @@ -use crate::local_logger::suspend_progress_bar; -use crate::prelude::*; -use crate::run::{ - config::Config, instruments::mongo_tracer::MongoTracer, - runner::helpers::ignored_objects_path::get_objects_path_to_ignore, - runner::helpers::introspected_node::setup_introspected_node, -}; -use lazy_static::lazy_static; -use std::fs::canonicalize; -use std::io::{Read, Write}; -use std::path::Path; -use std::process::ExitStatus; -use std::{collections::HashMap, env::consts::ARCH, process::Command}; -use std::{env, thread}; - -lazy_static! { - static ref BASE_INJECTED_ENV: HashMap<&'static str, String> = { - HashMap::from([ - ("PYTHONMALLOC", "malloc".into()), - ("PYTHONHASHSEED", "0".into()), - ("ARCH", ARCH.into()), - ("CODSPEED_ENV", "runner".into()), - ]) - }; - static ref VALGRIND_BASE_ARGS: Vec = { - let mut args = vec![]; - args.extend( - [ - "-q", - "--tool=callgrind", - "--trace-children=yes", - "--cache-sim=yes", - "--I1=32768,8,64", - "--D1=32768,8,64", - "--LL=8388608,16,64", - "--instr-atstart=no", - "--collect-systime=nsec", - "--compress-strings=no", - "--combine-dumps=yes", - "--dump-line=no", - ] - .iter() - .map(|x| x.to_string()), - ); - let children_skip_patterns = ["*esbuild"]; - args.push(format!( - "--trace-children-skip={}", - children_skip_patterns.join(",") - )); - args - }; -} - -fn get_bench_command(config: &Config) -> Result { - let bench_command = &config.command.trim(); - - if bench_command.is_empty() { - bail!("The bench command is empty"); - } - - Ok(bench_command - // Fixes a compatibility issue with cargo 1.66+ running directly under valgrind <3.20 - .replace("cargo codspeed", "cargo-codspeed")) -} - -pub const VALGRIND_EXECUTION_TARGET: &str = "valgrind::execution"; - -fn run_command_with_log_pipe(mut cmd: Command) -> Result { - fn log_tee( - mut reader: impl Read, - mut writer: impl Write, - log_prefix: Option<&str>, - ) -> Result<()> { - let prefix = log_prefix.unwrap_or(""); - let mut buffer = [0; 1024]; - loop { - let bytes_read = reader.read(&mut buffer)?; - if bytes_read == 0 { - break; - } - suspend_progress_bar(|| { - writer.write_all(&buffer[..bytes_read]).unwrap(); - trace!(target: VALGRIND_EXECUTION_TARGET, "{}{}", prefix, String::from_utf8_lossy(&buffer[..bytes_read])); - }); - } - Ok(()) - } - - let mut process = cmd - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn the process")?; - let stdout = process.stdout.take().expect("unable to get stdout"); - let stderr = process.stderr.take().expect("unable to get stderr"); - thread::spawn(move || { - log_tee(stdout, std::io::stdout(), None).unwrap(); - }); - thread::spawn(move || { - log_tee(stderr, std::io::stderr(), Some("[stderr]")).unwrap(); - }); - process.wait().context("failed to wait for the process") -} - -pub fn measure( - config: &Config, - profile_folder: &Path, - mongo_tracer: &Option, -) -> Result<()> { - debug!("profile dir: {}", profile_folder.display()); - - // Create the command - let mut cmd = Command::new("setarch"); - cmd.arg(ARCH).arg("-R"); - // Configure the environment - cmd.envs(BASE_INJECTED_ENV.iter()).env( - "PATH", - format!( - "{}:{}", - setup_introspected_node() - .map_err(|e| anyhow!("failed to setup NodeJS introspection. {}", e))? - .to_str() - .unwrap(), - env::var("PATH").unwrap_or_default(), - ), - ); - if let Some(cwd) = &config.working_directory { - let abs_cwd = canonicalize(cwd)?; - cmd.current_dir(abs_cwd); - } - // Configure valgrind - let profile_path = profile_folder.join("%p.out"); - let log_path = profile_folder.join("valgrind.log"); - cmd.arg("valgrind") - .args(VALGRIND_BASE_ARGS.iter()) - .args( - get_objects_path_to_ignore() - .iter() - .map(|x| format!("--obj-skip={}", x)), - ) - .arg(format!("--callgrind-out-file={}", profile_path.to_str().unwrap()).as_str()) - .arg(format!("--log-file={}", log_path.to_str().unwrap()).as_str()); - - // Set the command to execute - cmd.args(["sh", "-c", get_bench_command(config)?.as_str()]); - - // TODO: refactor and move this to the `Instrumentation` trait - if let Some(mongo_tracer) = mongo_tracer { - mongo_tracer.apply_run_command_transformations(&mut cmd)?; - } - - debug!("cmd: {:?}", cmd); - let status = run_command_with_log_pipe(cmd) - .map_err(|e| anyhow!("failed to execute the benchmark process. {}", e))?; - if !status.success() { - bail!("failed to execute the benchmark process"); - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_get_bench_command_empty() { - let config = Config::test(); - assert!(get_bench_command(&config).is_err()); - assert_eq!( - get_bench_command(&config).unwrap_err().to_string(), - "The bench command is empty" - ); - } - - #[test] - fn test_get_bench_command_cargo() { - let config = Config { - command: "cargo codspeed bench".into(), - ..Config::test() - }; - assert_eq!(get_bench_command(&config).unwrap(), "cargo-codspeed bench"); - } - - #[test] - fn test_get_bench_command_multiline() { - let config = Config { - // TODO: use indoc! macro - command: r#" -cargo codspeed bench --features "foo bar" -pnpm vitest bench "my-app" -pytest tests/ --codspeed -"# - .into(), - ..Config::test() - }; - assert_eq!( - get_bench_command(&config).unwrap(), - r#"cargo-codspeed bench --features "foo bar" -pnpm vitest bench "my-app" -pytest tests/ --codspeed"# - ); - } -} diff --git a/src/run/runner/valgrind/executor.rs b/src/run/runner/valgrind/executor.rs new file mode 100644 index 00000000..c2bbdb12 --- /dev/null +++ b/src/run/runner/valgrind/executor.rs @@ -0,0 +1,55 @@ +use async_trait::async_trait; + +use crate::prelude::*; +use crate::run::instruments::mongo_tracer::MongoTracer; +use crate::run::runner::executor::Executor; +use crate::run::runner::{ExecutorName, RunData}; +use crate::run::{check_system::SystemInfo, config::Config}; + +use super::{helpers::perf_maps::harvest_perf_maps, measure, setup::setup}; + +pub const INSTRUMENTATION_RUNNER_MODE: &str = "instrumentation"; + +pub struct ValgrindExecutor; + +#[async_trait(?Send)] +impl Executor for ValgrindExecutor { + fn name(&self) -> ExecutorName { + ExecutorName::Valgrind + } + + async fn setup( + &self, + config: &Config, + system_info: &SystemInfo, + _run_data: &RunData, + ) -> Result<()> { + setup(system_info, config).await?; + + Ok(()) + } + + async fn run( + &self, + config: &Config, + _system_info: &SystemInfo, + run_data: &RunData, + mongo_tracer: &Option, + ) -> Result<()> { + //TODO: add valgrind version check + measure::measure(config, &run_data.profile_folder, mongo_tracer)?; + + Ok(()) + } + + async fn teardown( + &self, + _config: &Config, + _system_info: &SystemInfo, + run_data: &RunData, + ) -> Result<()> { + harvest_perf_maps(&run_data.profile_folder)?; + + Ok(()) + } +} diff --git a/src/run/runner/helpers/download_file.rs b/src/run/runner/valgrind/helpers/download_file.rs similarity index 100% rename from src/run/runner/helpers/download_file.rs rename to src/run/runner/valgrind/helpers/download_file.rs diff --git a/src/run/runner/helpers/ignored_objects_path.rs b/src/run/runner/valgrind/helpers/ignored_objects_path.rs similarity index 100% rename from src/run/runner/helpers/ignored_objects_path.rs rename to src/run/runner/valgrind/helpers/ignored_objects_path.rs diff --git a/src/run/runner/helpers/introspected_node/mod.rs b/src/run/runner/valgrind/helpers/introspected_nodejs/mod.rs similarity index 84% rename from src/run/runner/helpers/introspected_node/mod.rs rename to src/run/runner/valgrind/helpers/introspected_nodejs/mod.rs index 42302a9d..047bc623 100644 --- a/src/run/runner/helpers/introspected_node/mod.rs +++ b/src/run/runner/valgrind/helpers/introspected_nodejs/mod.rs @@ -3,9 +3,9 @@ use std::{env, fs::File, io::Write, os::unix::fs::PermissionsExt, path::PathBuf} const INTROSPECTED_NODE_SCRIPT: &str = include_str!("node.sh"); -/// Creates the node script that will replace the node binary while running +/// Creates the `node` script that will replace the `node` binary while running /// Returns the path to the script folder, which should be added to the PATH environment variable -pub fn setup_introspected_node() -> Result { +pub fn setup_introspected_nodejs() -> Result { let script_folder = env::temp_dir().join("codspeed_introspected_node"); std::fs::create_dir_all(&script_folder)?; let script_path = script_folder.join("node"); diff --git a/src/run/runner/helpers/introspected_node/node.sh b/src/run/runner/valgrind/helpers/introspected_nodejs/node.sh similarity index 100% rename from src/run/runner/helpers/introspected_node/node.sh rename to src/run/runner/valgrind/helpers/introspected_nodejs/node.sh diff --git a/src/run/runner/valgrind/helpers/mod.rs b/src/run/runner/valgrind/helpers/mod.rs new file mode 100644 index 00000000..9072b60a --- /dev/null +++ b/src/run/runner/valgrind/helpers/mod.rs @@ -0,0 +1,4 @@ +pub mod download_file; +pub mod ignored_objects_path; +pub mod introspected_nodejs; +pub mod perf_maps; diff --git a/src/run/runner/helpers/perf_maps.rs b/src/run/runner/valgrind/helpers/perf_maps.rs similarity index 100% rename from src/run/runner/helpers/perf_maps.rs rename to src/run/runner/valgrind/helpers/perf_maps.rs diff --git a/src/run/runner/valgrind/measure.rs b/src/run/runner/valgrind/measure.rs new file mode 100644 index 00000000..a0dd5074 --- /dev/null +++ b/src/run/runner/valgrind/measure.rs @@ -0,0 +1,99 @@ +use crate::prelude::*; +use crate::run::runner::helpers::env::BASE_INJECTED_ENV; +use crate::run::runner::helpers::get_bench_command::get_bench_command; +use crate::run::runner::helpers::run_command_with_log_pipe::run_command_with_log_pipe; +use crate::run::runner::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; +use crate::run::runner::valgrind::helpers::introspected_nodejs::setup_introspected_nodejs; +use crate::run::{config::Config, instruments::mongo_tracer::MongoTracer}; +use lazy_static::lazy_static; +use std::env; +use std::fs::canonicalize; +use std::path::Path; +use std::{env::consts::ARCH, process::Command}; + +lazy_static! { + static ref VALGRIND_BASE_ARGS: Vec = { + let mut args = vec![]; + args.extend( + [ + "-q", + "--tool=callgrind", + "--trace-children=yes", + "--cache-sim=yes", + "--I1=32768,8,64", + "--D1=32768,8,64", + "--LL=8388608,16,64", + "--instr-atstart=no", + "--collect-systime=nsec", + "--compress-strings=no", + "--combine-dumps=yes", + "--dump-line=no", + ] + .iter() + .map(|x| x.to_string()), + ); + let children_skip_patterns = ["*esbuild"]; + args.push(format!( + "--trace-children-skip={}", + children_skip_patterns.join(",") + )); + args + }; +} + +pub const VALGRIND_EXECUTION_TARGET: &str = "valgrind::execution"; + +pub fn measure( + config: &Config, + profile_folder: &Path, + mongo_tracer: &Option, +) -> Result<()> { + // Create the command + let mut cmd = Command::new("setarch"); + cmd.arg(ARCH).arg("-R"); + // Configure the environment + cmd.envs(BASE_INJECTED_ENV.iter()).env( + "PATH", + format!( + "{}:{}", + setup_introspected_nodejs() + .map_err(|e| anyhow!("failed to setup NodeJS introspection. {}", e))? + .to_str() + .unwrap(), + env::var("PATH").unwrap_or_default(), + ), + ); + if let Some(cwd) = &config.working_directory { + let abs_cwd = canonicalize(cwd)?; + cmd.current_dir(abs_cwd); + } + // Configure valgrind + let profile_path = profile_folder.join("%p.out"); + let log_path = profile_folder.join("valgrind.log"); + cmd.arg("valgrind") + .args(VALGRIND_BASE_ARGS.iter()) + .args( + get_objects_path_to_ignore() + .iter() + .map(|x| format!("--obj-skip={}", x)), + ) + .arg(format!("--callgrind-out-file={}", profile_path.to_str().unwrap()).as_str()) + .arg(format!("--log-file={}", log_path.to_str().unwrap()).as_str()); + + // Set the command to execute + cmd.args(["sh", "-c", get_bench_command(config)?.as_str()]); + + // TODO: refactor and move this to the `Instruments` struct + if let Some(mongo_tracer) = mongo_tracer { + mongo_tracer.apply_run_command_transformations(&mut cmd)?; + } + + debug!("cmd: {:?}", cmd); + let status = run_command_with_log_pipe(cmd, VALGRIND_EXECUTION_TARGET) + .map_err(|e| anyhow!("failed to execute the benchmark process. {}", e))?; + if !status.success() { + bail!("failed to execute the benchmark process"); + } + + Ok(()) +} diff --git a/src/run/runner/valgrind/mod.rs b/src/run/runner/valgrind/mod.rs new file mode 100644 index 00000000..0632d063 --- /dev/null +++ b/src/run/runner/valgrind/mod.rs @@ -0,0 +1,6 @@ +pub mod executor; +mod helpers; +mod measure; +mod setup; + +pub use measure::VALGRIND_EXECUTION_TARGET; diff --git a/src/run/runner/setup.rs b/src/run/runner/valgrind/setup.rs similarity index 96% rename from src/run/runner/setup.rs rename to src/run/runner/valgrind/setup.rs index 37835a3d..dfe5dadd 100644 --- a/src/run/runner/setup.rs +++ b/src/run/runner/valgrind/setup.rs @@ -129,6 +129,7 @@ async fn install_mongodb_tracer() -> Result<()> { pub async fn setup(system_info: &SystemInfo, config: &Config) -> Result<()> { install_valgrind(system_info).await?; + // TODO: move into setup of the Instruments struct if config.instruments.is_mongodb_enabled() { install_mongodb_tracer().await?; } @@ -149,8 +150,7 @@ mod tests { os: "Ubuntu".to_string(), os_version: "22.04".to_string(), arch: "x86_64".to_string(), - host: "host".to_string(), - user: "user".to_string(), + ..SystemInfo::test() }; assert_snapshot!( get_codspeed_valgrind_filename(&system_info).unwrap(), @@ -164,8 +164,7 @@ mod tests { os: "Debian".to_string(), os_version: "11".to_string(), arch: "x86_64".to_string(), - host: "host".to_string(), - user: "user".to_string(), + ..SystemInfo::test() }; assert_snapshot!( get_codspeed_valgrind_filename(&system_info).unwrap(), @@ -179,8 +178,7 @@ mod tests { os: "Ubuntu".to_string(), os_version: "22.04".to_string(), arch: "aarch64".to_string(), - host: "host".to_string(), - user: "user".to_string(), + ..SystemInfo::test() }; assert_snapshot!( get_codspeed_valgrind_filename(&system_info).unwrap(), diff --git a/src/run/runner/wall_time/executor.rs b/src/run/runner/wall_time/executor.rs new file mode 100644 index 00000000..3388c632 --- /dev/null +++ b/src/run/runner/wall_time/executor.rs @@ -0,0 +1,75 @@ +use crate::prelude::*; + +use crate::run::instruments::mongo_tracer::MongoTracer; +use crate::run::runner::executor::Executor; +use crate::run::runner::helpers::env::BASE_INJECTED_ENV; +use crate::run::runner::helpers::get_bench_command::get_bench_command; +use crate::run::runner::helpers::run_command_with_log_pipe::run_command_with_log_pipe; +use crate::run::runner::{ExecutorName, RunData}; +use crate::run::{check_system::SystemInfo, config::Config}; +use async_trait::async_trait; +use std::fs::canonicalize; +use std::process::Command; + +const WALL_TIME_EXECUTION_TARGET: &str = "walltime::execution"; +pub const WALL_TIME_RUNNER_MODE: &str = "walltime"; + +pub struct WallTimeExecutor; + +#[async_trait(?Send)] +impl Executor for WallTimeExecutor { + fn name(&self) -> ExecutorName { + ExecutorName::WallTime + } + + async fn setup( + &self, + _config: &Config, + _system_info: &SystemInfo, + _run_data: &RunData, + ) -> Result<()> { + Ok(()) + } + + async fn run( + &self, + config: &Config, + _system_info: &SystemInfo, + run_data: &RunData, + _mongo_tracer: &Option, + ) -> Result<()> { + let mut cmd = Command::new("sh"); + + cmd.env("CODSPEED_RUNNER_MODE", &self.name().to_string()) + .env( + "CODSPEED_PROFILE_FOLDER", + run_data.profile_folder.to_str().unwrap(), + ) + .envs(BASE_INJECTED_ENV.iter()); + + if let Some(cwd) = &config.working_directory { + let abs_cwd = canonicalize(cwd)?; + cmd.current_dir(abs_cwd); + } + + cmd.args(["-c", get_bench_command(config)?.as_str()]); + + debug!("cmd: {:?}", cmd); + let status = run_command_with_log_pipe(cmd, WALL_TIME_EXECUTION_TARGET) + .map_err(|e| anyhow!("failed to execute the benchmark process. {}", e))?; + if !status.success() { + bail!("failed to execute the benchmark process"); + } + + Ok(()) + } + + async fn teardown( + &self, + _config: &Config, + _system_info: &SystemInfo, + _run_data: &RunData, + ) -> Result<()> { + Ok(()) + } +} diff --git a/src/run/runner/wall_time/mod.rs b/src/run/runner/wall_time/mod.rs new file mode 100644 index 00000000..0c95fdab --- /dev/null +++ b/src/run/runner/wall_time/mod.rs @@ -0,0 +1 @@ +pub mod executor; diff --git a/src/run/uploader/interfaces.rs b/src/run/uploader/interfaces.rs index 72423d8d..bc56bbd5 100644 --- a/src/run/uploader/interfaces.rs +++ b/src/run/uploader/interfaces.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use crate::run::{ check_system::SystemInfo, ci_provider::interfaces::ProviderMetadata, - instruments::InstrumentNames, + instruments::InstrumentName, runner::ExecutorName, }; #[derive(Deserialize, Serialize, Debug)] @@ -23,7 +23,8 @@ pub struct UploadMetadata { pub struct Runner { pub name: String, pub version: String, - pub instruments: Vec, + pub instruments: Vec, + pub executor: ExecutorName, #[serde(flatten)] pub system_info: SystemInfo, } diff --git a/src/run/uploader/snapshots/codspeed__run__uploader__upload_metadata__tests__get_metadata_hash.snap b/src/run/uploader/snapshots/codspeed__run__uploader__upload_metadata__tests__get_metadata_hash.snap index 118e8881..172ff56d 100644 --- a/src/run/uploader/snapshots/codspeed__run__uploader__upload_metadata__tests__get_metadata_hash.snap +++ b/src/run/uploader/snapshots/codspeed__run__uploader__upload_metadata__tests__get_metadata_hash.snap @@ -3,7 +3,7 @@ source: src/run/uploader/upload_metadata.rs expression: upload_metadata --- { - "version": 3, + "version": 4, "tokenless": true, "profileMd5": "jp/k05RKuqP3ERQuIIvx4Q==", "runner": { @@ -12,11 +12,17 @@ expression: upload_metadata "instruments": [ "MongoDB" ], + "executor": "valgrind", "os": "Ubuntu", "osVersion": "20.04", "arch": "x86_64", "host": "host", - "user": "user" + "user": "user", + "cpuBrand": "Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz", + "cpuName": "cpu0", + "cpuVendorId": "GenuineIntel", + "cpuCores": 2, + "totalMemoryGb": 8 }, "platform": "github-actions", "commitHash": "5bd77cb0da72bef094893ed45fb793ff16ecfbe3", diff --git a/src/run/uploader/upload.rs b/src/run/uploader/upload.rs index 3cbaf780..a36d86d2 100644 --- a/src/run/uploader/upload.rs +++ b/src/run/uploader/upload.rs @@ -1,3 +1,4 @@ +use crate::run::runner::ExecutorName; use crate::run::{ check_system::SystemInfo, ci_provider::CIProvider, config::Config, runner::RunData, uploader::UploadError, @@ -99,12 +100,14 @@ pub async fn upload( system_info: &SystemInfo, provider: &Box, run_data: &RunData, + executor_name: ExecutorName, ) -> Result { let (archive_buffer, archive_hash) = get_profile_archive_buffer(run_data).await?; debug!("CI provider detected: {:#?}", provider.get_provider_name()); - let upload_metadata = provider.get_upload_metadata(config, system_info, &archive_hash)?; + let upload_metadata = + provider.get_upload_metadata(config, system_info, &archive_hash, executor_name)?; debug!("Upload metadata: {:#?}", upload_metadata); info!( "Linked repository: {}\n", @@ -188,9 +191,15 @@ mod tests { ], async { let provider = crate::run::ci_provider::get_provider(&config).unwrap(); - upload(&config, &system_info, &provider, &run_data) - .await - .unwrap(); + upload( + &config, + &system_info, + &provider, + &run_data, + ExecutorName::Valgrind, + ) + .await + .unwrap(); }, ) .await; diff --git a/src/run/uploader/upload_metadata.rs b/src/run/uploader/upload_metadata.rs index 081bfa23..4f49d48e 100644 --- a/src/run/uploader/upload_metadata.rs +++ b/src/run/uploader/upload_metadata.rs @@ -16,20 +16,22 @@ mod tests { use crate::run::{ check_system::SystemInfo, ci_provider::interfaces::{GhData, ProviderMetadata, RunEvent, Sender}, - instruments::InstrumentNames, + instruments::InstrumentName, + runner::ExecutorName, uploader::{Runner, UploadMetadata}, }; #[test] fn test_get_metadata_hash() { let upload_metadata = UploadMetadata { - version: Some(3), + version: Some(4), tokenless: true, profile_md5: "jp/k05RKuqP3ERQuIIvx4Q==".into(), runner: Runner { name: "codspeed-runner".into(), version: "2.1.0".into(), - instruments: vec![InstrumentNames::MongoDB], + instruments: vec![InstrumentName::MongoDB], + executor: ExecutorName::Valgrind, system_info: SystemInfo::test(), }, platform: "github-actions".into(), @@ -56,7 +58,7 @@ mod tests { let hash = upload_metadata.get_hash(); assert_eq!( hash, - "ada5057b0c440844a1558eed80a1993a41756984cc6147fdef459ce8a289f1d7" + "3fc8e6c1b3f4adfe00e027d148f308a34730f1f2fa053c19378ce3e97b5dbab1" ); assert_json_snapshot!(upload_metadata); }