feat!(cargo-codspeed): Allow filtering by benchmark name

.github/workflows/ci.yml

@@ -87,6 +87,7 @@ jobs:
           MY_ENV_VAR: "YES"
         with:
           run: cargo codspeed run
+          mode: instrumentation
           token: ${{ secrets.CODSPEED_TOKEN }}
 
   compat-integration-test-walltime:
@@ -116,9 +117,9 @@ jobs:
         uses: CodSpeedHQ/action@main
         env:
           MY_ENV_VAR: "YES"
-          CODSPEED_PERF_ENABLED: true
         with:
           run: cargo codspeed run
+          mode: walltime
           token: ${{ secrets.CODSPEED_TOKEN }}
 
   check:

crates/cargo-codspeed/src/app.rs

@@ -3,7 +3,7 @@ use cargo_metadata::MetadataCommand;
 use clap::{Args, Parser, Subcommand};
 use std::{ffi::OsString, process::exit};
 
-use crate::build::build_benches;
+use crate::build::{build_benches, BuildConfig};
 
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
@@ -21,58 +21,70 @@ struct Cli {
     command: Commands,
 }
 
+const PACKAGE_HELP: &str = "Package Selection";
 #[derive(Args)]
 pub(crate) struct PackageFilters {
     /// Select all packages in the workspace
-    #[arg(long)]
+    #[arg(long, help_heading = PACKAGE_HELP)]
     pub(crate) workspace: bool,
-    /// Exclude packages
-    #[arg(long)]
+    /// Package to exclude
+    #[arg(long, value_name = "SPEC", help_heading = PACKAGE_HELP)]
     pub(crate) exclude: Vec<String>,
-    /// Package to select (builds all workspace package by default)
-    #[arg(short, long)]
+    /// Package to select
+    #[arg(short, long, value_name= "SPEC", help_heading = PACKAGE_HELP)]
     pub(crate) package: Vec<String>,
 }
 
 #[derive(Args)]
-pub(crate) struct Filters {
-    /// Optional list of benchmarks to build (builds all benchmarks by default)
+pub(crate) struct BenchTargetFilters {
+    /// Select only the specified benchmark target (all benchmark targets by default)
+    #[arg(long, help_heading = TARGET_HELP)]
     pub(crate) bench: Option<Vec<String>>,
-    #[command(flatten)]
-    pub(crate) package: PackageFilters,
 }
 
+const FEATURE_HELP: &str = "Feature Selection";
+const COMPILATION_HELP: &str = "Compilation Options";
+const TARGET_HELP: &str = "Target Selection";
 #[derive(Subcommand)]
 enum Commands {
     /// Build the benchmarks
     Build {
         #[command(flatten)]
-        filters: Filters,
+        package_filters: PackageFilters,
 
         /// Space or comma separated list of features to activate
-        #[arg(short = 'F', long)]
+        #[arg(short = 'F', long, help_heading = FEATURE_HELP)]
         features: Option<String>,
 
         /// Activate all available features of all selected packages.
-        #[arg(long)]
+        #[arg(long, help_heading = FEATURE_HELP)]
         all_features: bool,
 
         /// Do not activate the `default` feature of the selected packages.
-        #[arg(long)]
+        #[arg(long, help_heading = FEATURE_HELP)]
         no_default_features: bool,
 
         /// Number of parallel jobs, defaults to # of CPUs.
-        #[arg(short, long)]
+        #[arg(short, long, help_heading = COMPILATION_HELP)]
         jobs: Option<u32>,
 
         /// Build the benchmarks with the specified profile
-        #[arg(long, default_value = "bench")]
+        #[arg(long, default_value = "bench", help_heading = COMPILATION_HELP)]
         profile: String,
+
+        #[command(flatten)]
+        bench_target_filters: BenchTargetFilters,
     },
     /// Run the previously built benchmarks
     Run {
+        /// If specified, only run benches containing this string in their names
+        benchname: Option<String>,
+
+        #[command(flatten)]
+        package_filters: PackageFilters,
+
         #[command(flatten)]
-        filters: Filters,
+        bench_target_filters: BenchTargetFilters,
     },
 }
 
@@ -85,7 +97,8 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
 
     let res = match cli.command {
         Commands::Build {
-            filters,
+            package_filters,
+            bench_target_filters,
             features,
             all_features,
             jobs,
@@ -113,15 +126,28 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
                 features.map(|f| f.split([' ', ',']).map(|s| s.to_string()).collect_vec());
             build_benches(
                 &metadata,
-                filters,
-                features,
-                profile,
-                cli.quiet,
-                measurement_mode,
-                passthrough_flags,
+                BuildConfig {
+                    package_filters,
+                    bench_target_filters,
+                    features,
+                    profile,
+                    quiet: cli.quiet,
+                    measurement_mode,
+                    passthrough_flags,
+                },
             )
         }
-        Commands::Run { filters } => run_benches(&metadata, filters, measurement_mode),
+        Commands::Run {
+            benchname,
+            package_filters,
+            bench_target_filters,
+        } => run_benches(
+            &metadata,
+            benchname,
+            package_filters,
+            bench_target_filters,
+            measurement_mode,
+        ),
     };
 
     if let Err(e) = res {

crates/cargo-codspeed/src/build.rs

@@ -1,5 +1,5 @@
 use crate::{
-    app::{Filters, PackageFilters},
+    app::{BenchTargetFilters, PackageFilters},
     helpers::{clear_dir, get_codspeed_target_dir},
     measurement_mode::MeasurementMode,
     prelude::*,
@@ -8,7 +8,8 @@ use cargo_metadata::{camino::Utf8PathBuf, Message, Metadata, TargetKind};
 use std::process::{exit, Command, Stdio};
 
 struct BuildOptions<'a> {
-    filters: Filters,
+    bench_target_filters: BenchTargetFilters,
+    package_filters: PackageFilters,
     features: &'a Option<Vec<String>>,
     profile: &'a str,
     passthrough_flags: &'a Vec<String>,
@@ -20,6 +21,16 @@ struct BuiltBench {
     executable_path: Utf8PathBuf,
 }
 
+pub struct BuildConfig {
+    pub package_filters: PackageFilters,
+    pub bench_target_filters: BenchTargetFilters,
+    pub features: Option<Vec<String>>,
+    pub profile: String,
+    pub quiet: bool,
+    pub measurement_mode: MeasurementMode,
+    pub passthrough_flags: Vec<String>,
+}
+
 impl BuildOptions<'_> {
     /// Builds the benchmarks by invoking cargo
     /// Returns a list of built benchmarks, with path to associated executables
@@ -49,6 +60,19 @@ impl BuildOptions<'_> {
         );
 
         let mut built_benches = Vec::new();
+
+        let package_names = self
+            .package_filters
+            .packages_from_flags(metadata)
+            .map_err(|e| {
+                // Avoid leaving an orphan cargo process, even if something went wrong
+                cargo.wait().expect("Could not get cargo's exist status");
+                e
+            })?
+            .into_iter()
+            .map(|t| t.name.clone())
+            .collect::<Vec<_>>();
+
         for message in Message::parse_stream(reader) {
             match message.expect("Failed to parse message") {
                 // Those messages will include build errors and warnings even if stderr also contain some of them
@@ -66,19 +90,14 @@ impl BuildOptions<'_> {
                         .find(|p| p.id == artifact.package_id)
                         .expect("Could not find package");
 
-                    let bench_name = artifact.target.name;
+                    let bench_target_name = artifact.target.name;
 
-                    let add_bench_to_codspeed_dir = match &self.filters.bench {
-                        Some(allowed_bench_names) => allowed_bench_names
-                            .iter()
-                            .any(|allowed_bench_name| bench_name.contains(allowed_bench_name)),
-                        None => true,
-                    };
+                    let add_bench_to_codspeed_dir = package_names.iter().contains(&package.name);
 
                     if add_bench_to_codspeed_dir {
                         built_benches.push(BuiltBench {
                             package: package.name.clone(),
-                            bench: bench_name,
+                            bench: bench_target_name,
                             executable_path: artifact
                                 .executable
                                 .expect("Unexpected missing executable path"),
@@ -159,7 +178,15 @@ impl BuildOptions<'_> {
     /// This command explicitly ignores the `self.benches`: all benches are built
     fn build_command(&self, measurement_mode: MeasurementMode) -> Command {
         let mut cargo = Command::new("cargo");
-        cargo.args(["build", "--benches"]);
+        cargo.arg("build");
+
+        if let Some(bench_target_filters) = &self.bench_target_filters.bench {
+            for bench_target_filter in bench_target_filters {
+                cargo.args(["--bench", bench_target_filter]);
+            }
+        } else {
+            cargo.args(["--benches"]);
+        }
 
         self.add_rust_flags(&mut cargo, measurement_mode);
 
@@ -171,7 +198,7 @@ impl BuildOptions<'_> {
 
         cargo.arg("--profile").arg(self.profile);
 
-        self.filters.package.add_cargo_args(&mut cargo);
+        self.package_filters.add_cargo_args(&mut cargo);
 
         cargo
     }
@@ -197,22 +224,15 @@ impl PackageFilters {
     }
 }
 
-pub fn build_benches(
-    metadata: &Metadata,
-    filters: Filters,
-    features: Option<Vec<String>>,
-    profile: String,
-    quiet: bool,
-    measurement_mode: MeasurementMode,
-    passthrough_flags: Vec<String>,
-) -> Result<()> {
+pub fn build_benches(metadata: &Metadata, config: BuildConfig) -> Result<()> {
     let built_benches = BuildOptions {
-        filters,
-        features: &features,
-        profile: &profile,
-        passthrough_flags: &passthrough_flags,
+        bench_target_filters: config.bench_target_filters,
+        package_filters: config.package_filters,
+        features: &config.features,
+        profile: &config.profile,
+        passthrough_flags: &config.passthrough_flags,
     }
-    .build(metadata, quiet, measurement_mode)?;
+    .build(metadata, config.quiet, config.measurement_mode)?;
 
     if built_benches.is_empty() {
         bail!(
@@ -221,7 +241,7 @@ pub fn build_benches(
         );
     }
 
-    let codspeed_target_dir = get_codspeed_target_dir(metadata, measurement_mode);
+    let codspeed_target_dir = get_codspeed_target_dir(metadata, config.measurement_mode);
     let built_bench_count = built_benches.len();
 
     // Create and clear packages codspeed target directories