diff --git a/database/src/lib.rs b/database/src/lib.rs index 053e941c2..bd34e1b2e 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -232,6 +232,17 @@ impl Profile { pub fn default_profiles() -> Vec { vec![Profile::Check, Profile::Debug, Profile::Doc, Profile::Opt] } + + pub fn all_values() -> &'static [Self] { + &[ + Self::Check, + Self::Debug, + Self::Opt, + Self::Doc, + Self::DocJson, + Self::Clippy, + ] + } } impl std::str::FromStr for Profile { @@ -1023,11 +1034,7 @@ impl BenchmarkRequest { return Ok(Profile::default_profiles()); } - self.profiles - .split(',') - .map(Profile::from_str) - .collect::, _>>() - .map_err(|e| anyhow::anyhow!("Invalid profile: {e}")) + parse_profiles(&self.profiles).map_err(|e| anyhow::anyhow!("{e}")) } pub fn is_completed(&self) -> bool { @@ -1056,6 +1063,13 @@ pub fn parse_backends(backends: &str) -> Result, String> { .collect() } +pub fn parse_profiles(profiles: &str) -> Result, String> { + profiles + .split(',') + .map(|s| Profile::from_str(s).map_err(|_| format!("Invalid profile: {s}"))) + .collect() +} + /// Cached information about benchmark requests in the DB pub struct BenchmarkRequestIndex { /// Tags (SHA or release name) of all known benchmark requests diff --git a/site/frontend/templates/pages/help.html b/site/frontend/templates/pages/help.html index 2256c450c..4d662917c 100644 --- a/site/frontend/templates/pages/help.html +++ b/site/frontend/templates/pages/help.html @@ -21,10 +21,10 @@

@rust-timer commands

@rust-timer supports several commands, the most common (and simple) being @rust-timer queue. This command is usually invoked as @bors try @rust-timer - queue, - which starts a bors "try" run (not a merge). @rust-timer will wait for the try run - to finish, - and if it succeeds will then queue a perf run. + queue, which starts a bors "try build" (not a + merge). @rust-timer will wait for the try build to finish, and if it succeeds will + then queue a perf run.

@rust-timer queue has a few extra options that can be useful:

    @@ -49,6 +49,10 @@

    @rust-timer commands

    rustc-perf will also gather data for this backend for the parent/baseline commit, so that we have something to compare to. +
  • profiles=<PROFILES> configures which profiles should be benchmarked. + If you select a non-default profile, rustc-perf will also gather data for this profile for the + parent/baseline commit, so that we have something to compare to. +

@rust-timer build $commit will queue a perf run for the given commit $commit. diff --git a/site/src/request_handlers/github.rs b/site/src/request_handlers/github.rs index 4f86eb15c..c86d53ae2 100644 --- a/site/src/request_handlers/github.rs +++ b/site/src/request_handlers/github.rs @@ -6,7 +6,10 @@ use crate::github::{ use crate::job_queue::should_use_job_queue; use crate::load::SiteCtxt; -use database::{parse_backends, BenchmarkRequest, BenchmarkRequestInsertResult, CodegenBackend}; +use database::{ + parse_backends, parse_profiles, BenchmarkRequest, BenchmarkRequestInsertResult, CodegenBackend, + Profile, +}; use hashbrown::HashMap; use std::sync::Arc; @@ -93,8 +96,9 @@ async fn record_try_benchmark_request_without_artifacts( conn: &dyn database::pool::Connection, pr: u32, backends: &str, + profiles: &str, ) -> String { - let try_request = BenchmarkRequest::create_try_without_artifacts(pr, backends, ""); + let try_request = BenchmarkRequest::create_try_without_artifacts(pr, backends, profiles); log::info!("Inserting try benchmark request {try_request:?}"); match conn.insert_benchmark_request(&try_request).await { @@ -152,6 +156,7 @@ async fn handle_rust_timer( &*conn, issue.number, cmd.params.backends.unwrap_or(""), + cmd.params.profiles.unwrap_or(""), ) .await } else { @@ -200,6 +205,7 @@ async fn handle_rust_timer( &*conn, issue.number, command.params.backends.unwrap_or(""), + command.params.profiles.unwrap_or(""), ) .await; } else { @@ -277,6 +283,7 @@ fn parse_benchmark_parameters<'a>( exclude: args.remove("exclude").filter(|s| !s.is_empty()), runs: None, backends: args.remove("backends").filter(|s| !s.is_empty()), + profiles: args.remove("profiles").filter(|s| !s.is_empty()), }; if let Some(runs) = args.remove("runs").filter(|s| !s.is_empty()) { let Ok(runs) = runs.parse::() else { @@ -297,6 +304,19 @@ fn parse_benchmark_parameters<'a>( ) })?; } + if let Some(profiles) = ¶ms.profiles { + // Make sure that the profiles are correct + parse_profiles(profiles).map_err(|e| { + format!( + "Cannot parse profiles: {e}. Valid values are: {}", + Profile::all_values() + .iter() + .map(|b| b.as_str()) + .collect::>() + .join(", ") + ) + })?; + } if !args.is_empty() { Err(format!( @@ -346,6 +366,7 @@ struct BenchmarkParameters<'a> { exclude: Option<&'a str>, runs: Option, backends: Option<&'a str>, + profiles: Option<&'a str>, } pub async fn get_authorized_users() -> Result, String> { @@ -387,7 +408,7 @@ mod tests { #[test] fn build_command() { insta::assert_compact_debug_snapshot!(get_build_commands("@rust-timer build 5832462aa1d9373b24ace96ad2c50b7a18af9952"), - @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af9952", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } })]"#); + @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af9952", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } })]"#); } #[test] @@ -396,7 +417,7 @@ mod tests { @rust-timer build 5832462aa1d9373b24ace96ad2c50b7a18af9952 @rust-timer build 23936af287657fa4148aeab40cc2a780810fae52 "#), - @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af9952", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } }), Ok(BuildCommand { sha: "23936af287657fa4148aeab40cc2a780810fae52", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } })]"#); + @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af9952", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } }), Ok(BuildCommand { sha: "23936af287657fa4148aeab40cc2a780810fae52", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } })]"#); } #[test] @@ -408,14 +429,14 @@ mod tests { #[test] fn build_command_complex() { insta::assert_compact_debug_snapshot!(get_build_commands(" @rust-timer build sha123456 exclude=baz include=foo,bar runs=4"), - @r#"[Ok(BuildCommand { sha: "sha123456", params: BenchmarkParameters { include: Some("foo,bar"), exclude: Some("baz"), runs: Some(4), backends: None } })]"#); + @r#"[Ok(BuildCommand { sha: "sha123456", params: BenchmarkParameters { include: Some("foo,bar"), exclude: Some("baz"), runs: Some(4), backends: None, profiles: None } })]"#); } #[test] fn build_command_link() { insta::assert_compact_debug_snapshot!(get_build_commands(r#" @rust-timer build https://github.com/rust-lang/rust/commit/323f521bc6d8f2b966ba7838a3f3ee364e760b7e"#), - @r#"[Ok(BuildCommand { sha: "323f521bc6d8f2b966ba7838a3f3ee364e760b7e", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } })]"#); + @r#"[Ok(BuildCommand { sha: "323f521bc6d8f2b966ba7838a3f3ee364e760b7e", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } })]"#); } #[test] @@ -431,7 +452,7 @@ mod tests { #[test] fn queue_command() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue"), - @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } }))"); + @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } }))"); } #[test] @@ -449,19 +470,19 @@ mod tests { #[test] fn queue_command_include() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue include=abcd,feih"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("abcd,feih"), exclude: None, runs: None, backends: None } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("abcd,feih"), exclude: None, runs: None, backends: None, profiles: None } }))"#); } #[test] fn queue_command_exclude() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue exclude=foo134,barzbaz41baf"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: Some("foo134,barzbaz41baf"), runs: None, backends: None } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: Some("foo134,barzbaz41baf"), runs: None, backends: None, profiles: None } }))"#); } #[test] fn queue_command_runs() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue runs=5"), - @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: Some(5), backends: None } }))"); + @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: Some(5), backends: None, profiles: None } }))"); } #[test] @@ -473,7 +494,7 @@ mod tests { #[test] fn queue_command_combination() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue include=acda,13asd exclude=c13,DA runs=5"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("acda,13asd"), exclude: Some("c13,DA"), runs: Some(5), backends: None } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("acda,13asd"), exclude: Some("c13,DA"), runs: Some(5), backends: None, profiles: None } }))"#); } #[test] @@ -485,19 +506,19 @@ mod tests { #[test] fn queue_command_spaces() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue include=abcd,das "), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("abcd,das"), exclude: None, runs: None, backends: None } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("abcd,das"), exclude: None, runs: None, backends: None, profiles: None } }))"#); } #[test] fn queue_command_with_bors() { insta::assert_compact_debug_snapshot!(parse_queue_command("@bors try @rust-timer queue include=foo,bar"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("foo,bar"), exclude: None, runs: None, backends: None } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("foo,bar"), exclude: None, runs: None, backends: None, profiles: None } }))"#); } #[test] fn queue_command_parameter_order() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue runs=3 exclude=c,a include=b"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("b"), exclude: Some("c,a"), runs: Some(3), backends: None } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("b"), exclude: Some("c,a"), runs: Some(3), backends: None, profiles: None } }))"#); } #[test] @@ -508,7 +529,7 @@ Let's do a perf run quickly and then we can merge it. @bors try @rust-timer queue include=foo,bar Otherwise LGTM."#), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("foo,bar"), exclude: None, runs: None, backends: None } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("foo,bar"), exclude: None, runs: None, backends: None, profiles: None } }))"#); } fn get_build_commands(body: &str) -> Vec, String>> { @@ -518,44 +539,58 @@ Otherwise LGTM."#), #[test] fn build_command_with_backends() { insta::assert_compact_debug_snapshot!(get_build_commands(r#"@rust-timer build 5832462aa1d9373b24ace96ad2c50b7a18af995G"#), - @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995G", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } })]"#); + @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995G", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } })]"#); insta::assert_compact_debug_snapshot!(get_build_commands(r#"@rust-timer build 5832462aa1d9373b24ace96ad2c50b7a18af995A backends=Llvm"#), - @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995A", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Llvm") } })]"#); + @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995A", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Llvm"), profiles: None } })]"#); insta::assert_compact_debug_snapshot!(get_build_commands(r#"@rust-timer build 23936af287657fa4148aeab40cc2a780810fae5B backends=Cranelift"#), - @r#"[Ok(BuildCommand { sha: "23936af287657fa4148aeab40cc2a780810fae5B", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift") } })]"#); + @r#"[Ok(BuildCommand { sha: "23936af287657fa4148aeab40cc2a780810fae5B", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift"), profiles: None } })]"#); insta::assert_compact_debug_snapshot!(get_build_commands(r#"@rust-timer build 23936af287657fa4148aeab40cc2a780810fae5C backends=Cranelift,Llvm"#), - @r#"[Ok(BuildCommand { sha: "23936af287657fa4148aeab40cc2a780810fae5C", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift,Llvm") } })]"#); + @r#"[Ok(BuildCommand { sha: "23936af287657fa4148aeab40cc2a780810fae5C", params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift,Llvm"), profiles: None } })]"#); insta::assert_compact_debug_snapshot!(get_build_commands(r#"@rust-timer build 5832462aa1d9373b24ace96ad2c50b7a18af995D include=hello backends=Llvm"#), - @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995D", params: BenchmarkParameters { include: Some("hello"), exclude: None, runs: None, backends: Some("Llvm") } })]"#); + @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995D", params: BenchmarkParameters { include: Some("hello"), exclude: None, runs: None, backends: Some("Llvm"), profiles: None } })]"#); insta::assert_compact_debug_snapshot!(get_build_commands(r#"@rust-timer build 5832462aa1d9373b24ace96ad2c50b7a18af995E runs=10 backends=Llvm"#), - @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995E", params: BenchmarkParameters { include: None, exclude: None, runs: Some(10), backends: Some("Llvm") } })]"#); + @r#"[Ok(BuildCommand { sha: "5832462aa1d9373b24ace96ad2c50b7a18af995E", params: BenchmarkParameters { include: None, exclude: None, runs: Some(10), backends: Some("Llvm"), profiles: None } })]"#); } #[test] fn queue_command_with_backends() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue backends=Llvm"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Llvm") } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Llvm"), profiles: None } }))"#); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue backends=Cranelift"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift") } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift"), profiles: None } }))"#); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue backends=Cranelift,Llvm"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift,Llvm") } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift,Llvm"), profiles: None } }))"#); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue"), - @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } }))"); + @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } }))"); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue include=hello backends=Llvm"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("hello"), exclude: None, runs: None, backends: Some("Llvm") } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("hello"), exclude: None, runs: None, backends: Some("Llvm"), profiles: None } }))"#); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue include=hello exclude=ripgrep runs=3 backends=Llvm"), - @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("hello"), exclude: Some("ripgrep"), runs: Some(3), backends: Some("Llvm") } }))"#); + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: Some("hello"), exclude: Some("ripgrep"), runs: Some(3), backends: Some("Llvm"), profiles: None } }))"#); + } + + #[test] + fn queue_command_with_profiles() { + insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue profiles=Doc"), + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: Some("Doc") } }))"#); + insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue profiles=Check,Clippy"), + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: Some("Check,Clippy") } }))"#); + insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue profiles=Doc,Clippy,Opt backends=Cranelift,Llvm"), + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: Some("Cranelift,Llvm"), profiles: Some("Doc,Clippy,Opt") } }))"#); + insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue profiles=Foo"), + @r#"Some(Err("Cannot parse profiles: Invalid profile: Foo. Valid values are: check, debug, opt, doc, doc-json, clippy"))"#); + insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue profiles=check"), + @r#"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: Some("check") } }))"#); } #[test] fn no_empty_arguments_thank_you() { insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue include="), - @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } }))"); + @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } }))"); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue exclude="), - @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } }))"); + @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } }))"); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue runs="), - @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } }))"); + @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } }))"); insta::assert_compact_debug_snapshot!(parse_queue_command("@rust-timer queue backends="), - @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None } }))"); + @"Some(Ok(QueueCommand { params: BenchmarkParameters { include: None, exclude: None, runs: None, backends: None, profiles: None } }))"); } }