cod 177 run time based benchmarks in the codspeed runner

Cargo.toml

@@ -45,6 +45,7 @@ serde_yaml = "0.9.34"
 sysinfo = { version = "0.30.12", features = ["serde"] }
 indicatif = "0.17.8"
 console = "0.15.8"
+async-trait = "0.1.82"
 
 [dev-dependencies]
 temp-env = { version = "0.3.6", features = ["async_closure"] }

src/run/check_system.rs

@@ -3,7 +3,7 @@ use std::process::Command;
 use lazy_static::lazy_static;
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
-use sysinfo::System;
+use sysinfo::{CpuRefreshKind, MemoryRefreshKind, RefreshKind, System};
 
 use crate::prelude::*;
 
@@ -27,6 +27,11 @@ pub struct SystemInfo {
     pub arch: String,
     pub host: String,
     pub user: String,
+    pub cpu_brand: String,
+    pub cpu_name: String,
+    pub cpu_vendor_id: String,
+    pub cpu_cores: usize,
+    pub total_memory_gb: u64,
 }
 
 #[cfg(test)]
@@ -38,6 +43,11 @@ impl SystemInfo {
             arch: "x86_64".to_string(),
             host: "host".to_string(),
             user: "user".to_string(),
+            cpu_brand: "Intel(R) Xeon(R) CPU E5-2686 v4 @ 2.30GHz".to_string(),
+            cpu_name: "cpu0".to_string(),
+            cpu_vendor_id: "GenuineIntel".to_string(),
+            cpu_cores: 2,
+            total_memory_gb: 8,
         }
     }
 }
@@ -50,12 +60,37 @@ impl SystemInfo {
         let user = get_user()?;
         let host = System::host_name().ok_or(anyhow!("Failed to get host name"))?;
 
+        let s = System::new_with_specifics(
+            RefreshKind::new()
+                .with_cpu(CpuRefreshKind::everything())
+                .with_memory(MemoryRefreshKind::everything()),
+        );
+        let cpu_cores = s
+            .physical_core_count()
+            .ok_or(anyhow!("Failed to get CPU core count"))?;
+        let total_memory_gb = s.total_memory().div_ceil(1024_u64.pow(3));
+
+        // take the first CPU to get the brand, name and vendor id
+        let cpu = s
+            .cpus()
+            .iter()
+            .next()
+            .ok_or(anyhow!("Failed to get CPU info"))?;
+        let cpu_brand = cpu.brand().to_string();
+        let cpu_name = cpu.name().to_string();
+        let cpu_vendor_id = cpu.vendor_id().to_string();
+
         Ok(SystemInfo {
             os,
             os_version,
             arch,
             host,
             user,
+            cpu_brand,
+            cpu_name,
+            cpu_vendor_id,
+            cpu_cores,
+            total_memory_gb,
         })
     }
 }

src/run/ci_provider/provider.rs

@@ -4,6 +4,7 @@ use simplelog::SharedLogger;
 use crate::prelude::*;
 use crate::run::check_system::SystemInfo;
 use crate::run::config::Config;
+use crate::run::runner::ExecutorName;
 use crate::run::uploader::{Runner, UploadMetadata};
 
 use super::interfaces::ProviderMetadata;
@@ -77,13 +78,14 @@ pub trait CIProvider {
         config: &Config,
         system_info: &SystemInfo,
         archive_hash: &str,
+        executor_name: ExecutorName,
     ) -> Result<UploadMetadata> {
         let provider_metadata = self.get_provider_metadata()?;
 
         let commit_hash = get_commit_hash(&provider_metadata.repository_root_path)?;
 
         Ok(UploadMetadata {
-            version: Some(3),
+            version: Some(4),
             tokenless: config.token.is_none(),
             provider_metadata,
             profile_md5: archive_hash.into(),
@@ -92,6 +94,7 @@ pub trait CIProvider {
                 name: "codspeed-runner".into(),
                 version: crate::VERSION.into(),
                 instruments: config.instruments.get_active_instrument_names(),
+                executor: executor_name,
                 system_info: system_info.clone(),
             },
             platform: self.get_provider_slug().into(),

src/run/instruments/mod.rs

@@ -19,7 +19,7 @@ pub struct Instruments {
 }
 
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq, Hash)]
-pub enum InstrumentNames {
+pub enum InstrumentName {
     MongoDB,
 }
 
@@ -28,11 +28,11 @@ impl Instruments {
         self.mongodb.is_some()
     }
 
-    pub fn get_active_instrument_names(&self) -> Vec<InstrumentNames> {
+    pub fn get_active_instrument_names(&self) -> Vec<InstrumentName> {
         let mut names = vec![];
 
         if self.is_mongodb_enabled() {
-            names.push(InstrumentNames::MongoDB);
+            names.push(InstrumentName::MongoDB);
         }
 
         names
@@ -42,16 +42,16 @@ impl Instruments {
 impl TryFrom<&RunArgs> for Instruments {
     type Error = Error;
     fn try_from(args: &RunArgs) -> Result<Self> {
-        let mut validated_instrument_names: HashSet<InstrumentNames> = HashSet::new();
+        let mut validated_instrument_names: HashSet<InstrumentName> = HashSet::new();
 
         for instrument_name in &args.instruments {
             match instrument_name.as_str() {
-                "mongodb" => validated_instrument_names.insert(InstrumentNames::MongoDB),
+                "mongodb" => validated_instrument_names.insert(InstrumentName::MongoDB),
                 _ => bail!("Invalid instrument name: {}", instrument_name),
             };
         }
 
-        let mongodb = if validated_instrument_names.contains(&InstrumentNames::MongoDB) {
+        let mongodb = if validated_instrument_names.contains(&InstrumentName::MongoDB) {
             Some(MongoDBConfig {
                 uri_env_name: args.mongo_uri_env_name.clone(),
             })

src/run/instruments/mongo_tracer.rs

@@ -49,7 +49,7 @@ fn dump_tracer_log(mut stream: impl Read) -> Result<()> {
 }
 
 impl MongoTracer {
-    pub fn try_from(profile_folder: &PathBuf, mongodb_config: &MongoDBConfig) -> Result<Self> {
+    pub fn try_from(profile_folder: &Path, mongodb_config: &MongoDBConfig) -> Result<Self> {
         let user_input = match &mongodb_config.uri_env_name {
             Some(uri_env_name) => {
                 debug!(

src/run/mod.rs

@@ -5,6 +5,8 @@ use crate::run::{config::Config, logger::Logger};
 use crate::VERSION;
 use check_system::SystemInfo;
 use clap::Args;
+use instruments::mongo_tracer::MongoTracer;
+use runner::get_run_data;
 
 mod check_system;
 pub mod ci_provider;
@@ -114,12 +116,45 @@ pub async fn run(args: RunArgs, api_client: &CodSpeedAPIClient) -> Result<()> {
     let system_info = SystemInfo::new()?;
     check_system::check_system(&system_info)?;
 
-    let run_data = runner::run(&config, &system_info).await?;
+    let executor = runner::get_executor()?;
+
+    let run_data = get_run_data()?;
+
+    if !config.skip_setup {
+        start_group!("Preparing the environment");
+        executor.setup(&config, &system_info, &run_data).await?;
+        end_group!();
+    }
+
+    start_opened_group!("Running the benchmarks");
+
+    // TODO: refactor and move directly in the Instruments struct as a `start` method
+    let mongo_tracer = if let Some(mongodb_config) = &config.instruments.mongodb {
+        let mut mongo_tracer = MongoTracer::try_from(&run_data.profile_folder, mongodb_config)?;
+        mongo_tracer.start().await?;
+        Some(mongo_tracer)
+    } else {
+        None
+    };
+
+    executor
+        .run(&config, &system_info, &run_data, &mongo_tracer)
+        .await?;
+
+    // TODO: refactor and move directly in the Instruments struct as a `stop` method
+    if let Some(mut mongo_tracer) = mongo_tracer {
+        mongo_tracer.stop().await?;
+    }
+
+    executor.teardown(&config, &system_info, &run_data).await?;
+
+    end_group!();
 
     if !config.skip_upload {
         start_group!("Uploading performance data");
         logger.persist_log_to_profile_folder(&run_data)?;
-        let upload_result = uploader::upload(&config, &system_info, &provider, &run_data).await?;
+        let upload_result =
+            uploader::upload(&config, &system_info, &provider, &run_data, executor.name()).await?;
         end_group!();
 
         if provider.get_provider_slug() == "local" {

src/run/runner/executor.rs

@@ -0,0 +1,35 @@
+use async_trait::async_trait;
+
+use super::interfaces::{ExecutorName, RunData};
+use crate::prelude::*;
+use crate::run::instruments::mongo_tracer::MongoTracer;
+use crate::run::{check_system::SystemInfo, config::Config};
+
+#[async_trait(?Send)]
+pub trait Executor {
+    fn name(&self) -> ExecutorName;
+
+    async fn setup(
+        &self,
+        config: &Config,
+        system_info: &SystemInfo,
+        run_data: &RunData,
+    ) -> Result<()>;
+
+    /// Runs the executor
+    async fn run(
+        &self,
+        config: &Config,
+        system_info: &SystemInfo,
+        run_data: &RunData,
+        // TODO: use Instruments instead of directly passing the mongodb tracer
+        mongo_tracer: &Option<MongoTracer>,
+    ) -> Result<()>;
+
+    async fn teardown(
+        &self,
+        config: &Config,
+        system_info: &SystemInfo,
+        run_data: &RunData,
+    ) -> Result<()>;
+}

src/run/runner/helpers/env.rs

@@ -0,0 +1,14 @@
+use std::{collections::HashMap, env::consts::ARCH};
+
+use lazy_static::lazy_static;
+
+lazy_static! {
+    pub static ref BASE_INJECTED_ENV: HashMap<&'static str, String> = {
+        HashMap::from([
+            ("PYTHONMALLOC", "malloc".into()),
+            ("PYTHONHASHSEED", "0".into()),
+            ("ARCH", ARCH.into()),
+            ("CODSPEED_ENV", "runner".into()),
+        ])
+    };
+}

src/run/runner/helpers/get_bench_command.rs

@@ -0,0 +1,58 @@
+use crate::prelude::*;
+use crate::run::config::Config;
+
+pub fn get_bench_command(config: &Config) -> Result<String> {
+    let bench_command = &config.command.trim();
+
+    if bench_command.is_empty() {
+        bail!("The bench command is empty");
+    }
+
+    Ok(bench_command
+        // Fixes a compatibility issue with cargo 1.66+ running directly under valgrind <3.20
+        .replace("cargo codspeed", "cargo-codspeed"))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_get_bench_command_empty() {
+        let config = Config::test();
+        assert!(get_bench_command(&config).is_err());
+        assert_eq!(
+            get_bench_command(&config).unwrap_err().to_string(),
+            "The bench command is empty"
+        );
+    }
+
+    #[test]
+    fn test_get_bench_command_cargo() {
+        let config = Config {
+            command: "cargo codspeed bench".into(),
+            ..Config::test()
+        };
+        assert_eq!(get_bench_command(&config).unwrap(), "cargo-codspeed bench");
+    }
+
+    #[test]
+    fn test_get_bench_command_multiline() {
+        let config = Config {
+            // TODO: use indoc! macro
+            command: r#"
+cargo codspeed bench --features "foo bar"
+pnpm vitest bench "my-app"
+pytest tests/ --codspeed
+"#
+            .into(),
+            ..Config::test()
+        };
+        assert_eq!(
+            get_bench_command(&config).unwrap(),
+            r#"cargo-codspeed bench --features "foo bar"
+pnpm vitest bench "my-app"
+pytest tests/ --codspeed"#
+        );
+    }
+}

src/run/runner/helpers/mod.rs

@@ -1,5 +1,4 @@
-pub mod download_file;
-pub mod ignored_objects_path;
-pub mod introspected_node;
-pub mod perf_maps;
+pub mod env;
+pub mod get_bench_command;
 pub mod profile_folder;
+pub mod run_command_with_log_pipe;