From dc928cb734ec27b6727b8da4b4c7e8e3498a7ceb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 11 Nov 2025 10:58:56 +0100 Subject: [PATCH] Add stable benchmarks to the compile-time benchmark set --- .../src/benchmark_set/compile_benchmarks.rs | 14 ++++++- collector/src/benchmark_set/mod.rs | 10 ++++- collector/src/bin/collector.rs | 42 +++++++++---------- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/collector/src/benchmark_set/compile_benchmarks.rs b/collector/src/benchmark_set/compile_benchmarks.rs index add2344e7..e7b05ce4f 100644 --- a/collector/src/benchmark_set/compile_benchmarks.rs +++ b/collector/src/benchmark_set/compile_benchmarks.rs @@ -1,8 +1,20 @@ -//! This file contains an exhaustive list of all **non-stable** compile-time benchmarks +//! This file contains an exhaustive list of all compile-time benchmarks //! located in the `collector/compile-benchmarks` directory that are benchmarked in production. //! If new benchmarks are added/removed, they have to also be added/removed here, and in //! the [super::expand_benchmark_set] function. +// Stable benchmarks +pub(super) const CARGO: &str = "cargo"; +pub(super) const ENCODING: &str = "encoding"; +pub(super) const FUTURES: &str = "futures"; +pub(super) const HTML5EVER: &str = "html5ever"; +pub(super) const INFLATE: &str = "inflate"; +pub(super) const PISTON_IMAGE: &str = "piston-image"; +pub(super) const REGEX: &str = "regex"; +pub(super) const SYN: &str = "syn"; +pub(super) const TOKIO_WEBPUSH_SIMPLE: &str = "tokio-webpush-simple"; + +// Non-stable benchmarks pub(super) const AWAIT_CALL_TREE: &str = "await-call-tree"; pub(super) const BITMAPS_3_2_1: &str = "bitmaps-3.2.1"; pub(super) const BITMAPS_3_2_1_NEW_SOLVER: &str = "bitmaps-3.2.1-new-solver"; diff --git a/collector/src/benchmark_set/mod.rs b/collector/src/benchmark_set/mod.rs index baf521acc..90d8239c9 100644 --- a/collector/src/benchmark_set/mod.rs +++ b/collector/src/benchmark_set/mod.rs @@ -48,6 +48,7 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec { compile(AWAIT_CALL_TREE), compile(BITMAPS_3_2_1), compile(BITMAPS_3_2_1_NEW_SOLVER), + compile(CARGO), compile(CARGO_0_87_1), compile(CLAP_DERIVE_4_5_32), compile(COERCIONS), @@ -57,15 +58,19 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec { compile(DEEPLY_NESTED_MULTI), compile(DERIVE), compile(DIESEL_2_2_10), + compile(ENCODING), compile(EXTERNS), compile(EZA_0_21_2), + compile(FUTURES), compile(HELLOWORLD), compile(HELLOWORLD_TINY), + compile(HTML5EVER), compile(HTML5EVER_0_31_0), compile(HTML5EVER_0_31_0_NEW_SOLVER), compile(HYPER_1_6_0), compile(IMAGE_0_25_6), compile(INCLUDE_BLOB), + compile(INFLATE), compile(ISSUE_46449), compile(ISSUE_58319), compile(ISSUE_88862), @@ -75,7 +80,9 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec { compile(MATCH_STRESS), compile(NALGEBRA_0_33_0), compile(NALGEBRA_0_33_0_NEW_SOLVER), + compile(PISTON_IMAGE), compile(PROJECTION_CACHING), + compile(REGEX), compile(REGEX_AUTOMATA_0_4_8), compile(REGRESSION_31157), compile(RIPGREP_14_1_1), @@ -85,9 +92,11 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec { compile(SERDE_1_0_219_THREADS4), compile(SERDE_DERIVE_1_0_219), compile(STM32F4_0_15_1), + compile(SYN), compile(SYN_2_0_101), compile(SYN_2_0_101_NEW_SOLVER), compile(TOKEN_STREAM_STRESS), + compile(TOKIO_WEBPUSH_SIMPLE), compile(TT_MUNCHER), compile(TUPLE_STRESS), compile(TYPENUM_1_18_0), @@ -168,7 +177,6 @@ mod tests { get_compile_benchmarks(Path::new(BENCHMARK_DIR), CompileBenchmarkFilter::All) .unwrap() .into_iter() - .filter(|b| !b.category().is_stable()) .map(|b| b.name) .collect::>(); for benchmark in &all_compile_benchmarks { diff --git a/collector/src/bin/collector.rs b/collector/src/bin/collector.rs index f05bf9b0f..998158c38 100644 --- a/collector/src/bin/collector.rs +++ b/collector/src/bin/collector.rs @@ -1785,6 +1785,11 @@ async fn create_benchmark_configs( let mut bench_runtime = false; let mut bench_compile_benchmarks = HashSet::new(); + let is_release = match artifact_id { + ArtifactId::Commit(_) => false, + ArtifactId::Tag(_) => true, + }; + match job.kind() { database::BenchmarkJobKind::Runtime => { bench_runtime = true; @@ -1799,32 +1804,27 @@ async fn create_benchmark_configs( } } database::BenchmarkJobKind::Rustc => { - bench_rustc = true; + bench_rustc = !is_release; } } - let is_release = match artifact_id { - ArtifactId::Commit(_) => false, - ArtifactId::Tag(_) => true, - }; - if is_release { - bench_rustc = false; - bench_compile_benchmarks.retain(|benchmark| { - all_compile_benchmarks - .iter() - .find(|b| b.name == *benchmark) - .map(|b| b.category().is_stable()) - .unwrap_or(false) - }); - } + let benchmarks: Vec = all_compile_benchmarks + .iter() + .filter(|b| { + if !bench_compile_benchmarks.contains(&b.name) { + return false; + } + // Run stable benchmarks for releases and non-stable benchmarks for everything + // else. + let is_stable = b.category().is_stable(); + is_release == is_stable + }) + .cloned() + .collect(); - let compile_config = if bench_rustc || !bench_compile_benchmarks.is_empty() { + let compile_config = if bench_rustc || !benchmarks.is_empty() { Some(CompileBenchmarkConfig { - benchmarks: all_compile_benchmarks - .iter() - .filter(|b| bench_compile_benchmarks.contains(&b.name)) - .cloned() - .collect(), + benchmarks, profiles: vec![job.profile().into()], scenarios: Scenario::all(), backends: vec![job.backend().into()],