From fa80e9017cfff43434ff34b194101fe7bfe06e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 13 Oct 2025 14:08:44 +0200 Subject: [PATCH 1/4] Add kind column to job queue table constraint --- database/src/pool/postgres.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index 19fbb2e90..c52e676a2 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -420,6 +420,18 @@ static MIGRATIONS: &[&str] = &[ r#" ALTER TABLE job_queue ADD COLUMN kind TEXT NOT NULL DEFAULT 'compiletime'; "#, + r#" + ALTER TABLE job_queue DROP CONSTRAINT job_queue_unique; + ALTER TABLE job_queue ADD CONSTRAINT job_queue_unique + UNIQUE ( + request_tag, + target, + backend, + profile, + kind, + benchmark_set + ); + "#, ]; #[async_trait::async_trait] From 04da88a75f9e85a4394048a70407bcb6064b2e75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 13 Oct 2025 14:14:00 +0200 Subject: [PATCH 2/4] Error out if a job cannot be enqueue --- site/src/job_queue/mod.rs | 74 ++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index e0fb1d200..18e5a94c0 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -230,52 +230,68 @@ pub async fn enqueue_benchmark_request( let backends = request.backends()?; let profiles = request.profiles()?; + // Prevent the error from spamming the logs // let mut has_emitted_parent_sha_error = false; + let mut enqueue_job_required = async |request_tag, + target, + backend, + profile, + benchmark_set, + kind| { + let created_job = tx + .conn() + .enqueue_benchmark_job(request_tag, target, backend, profile, benchmark_set, kind) + .await?; + match created_job { + Some(_) => Ok(()), + None => Err(anyhow::anyhow!( + "Cannot created job for tag {request_tag} (target={target}, backend={backend}, profile={profile}, set={benchmark_set}, kind={kind}): job already exists in the DB" + )), + } + }; + // Enqueue Rustc job for only for x86_64 & llvm. This benchmark is how long // it takes to build the rust compiler. It takes a while to run and is // assumed that if the compilation of other rust project improve then this // too would improve. - tx.conn() - .enqueue_benchmark_job( + enqueue_job_required( + request_tag, + Target::X86_64UnknownLinuxGnu, + CodegenBackend::Llvm, + Profile::Opt, + 0u32, + BenchmarkJobKind::Rustc, + ) + .await?; + + // Target x benchmark_set x backend x profile -> BenchmarkJob + for target in Target::all() { + // Enqueue Runtime job for all targets using LLVM as the backend for + // runtime benchmarks + enqueue_job_required( request_tag, - Target::X86_64UnknownLinuxGnu, + target, CodegenBackend::Llvm, Profile::Opt, 0u32, - BenchmarkJobKind::Rustc, + BenchmarkJobKind::Runtime, ) .await?; - // Target x benchmark_set x backend x profile -> BenchmarkJob - for target in Target::all() { - // Enqueue Runtime job for all targets using LLVM as the backend for - // runtime benchmarks - tx.conn() - .enqueue_benchmark_job( - request_tag, - target, - CodegenBackend::Llvm, - Profile::Opt, - 0u32, - BenchmarkJobKind::Runtime, - ) - .await?; - for benchmark_set in 0..benchmark_set_count(target.into()) { for &backend in backends.iter() { for &profile in profiles.iter() { - tx.conn() - .enqueue_benchmark_job( - request_tag, - target, - backend, - profile, - benchmark_set as u32, - BenchmarkJobKind::Compiletime, - ) - .await?; + enqueue_job_required( + request_tag, + target, + backend, + profile, + benchmark_set as u32, + BenchmarkJobKind::Compiletime, + ) + .await?; // If there is a parent, we create a job for it too. The // database will ignore it if there is already a job there. // If the parent job has been deleted from the database From 63a0e4543a30e40f5b34d4c1a6506117b5664313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 13 Oct 2025 14:14:48 +0200 Subject: [PATCH 3/4] Enqueue runtime and rustc jobs last To first benchmark compile-time jobs. --- site/src/job_queue/mod.rs | 52 +++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index 18e5a94c0..939a84e16 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -252,34 +252,8 @@ pub async fn enqueue_benchmark_request( } }; - // Enqueue Rustc job for only for x86_64 & llvm. This benchmark is how long - // it takes to build the rust compiler. It takes a while to run and is - // assumed that if the compilation of other rust project improve then this - // too would improve. - enqueue_job_required( - request_tag, - Target::X86_64UnknownLinuxGnu, - CodegenBackend::Llvm, - Profile::Opt, - 0u32, - BenchmarkJobKind::Rustc, - ) - .await?; - // Target x benchmark_set x backend x profile -> BenchmarkJob for target in Target::all() { - // Enqueue Runtime job for all targets using LLVM as the backend for - // runtime benchmarks - enqueue_job_required( - request_tag, - target, - CodegenBackend::Llvm, - Profile::Opt, - 0u32, - BenchmarkJobKind::Runtime, - ) - .await?; - for benchmark_set in 0..benchmark_set_count(target.into()) { for &backend in backends.iter() { for &profile in profiles.iter() { @@ -331,8 +305,34 @@ pub async fn enqueue_benchmark_request( } } } + + // Enqueue Runtime job for all targets using LLVM as the backend for + // runtime benchmarks + enqueue_job_required( + request_tag, + target, + CodegenBackend::Llvm, + Profile::Opt, + 0u32, + BenchmarkJobKind::Runtime, + ) + .await?; } + // Enqueue Rustc job for only for x86_64 & llvm. This benchmark is how long + // it takes to build the rust compiler. It takes a while to run and is + // assumed that if the compilation of other rust project improve then this + // too would improve. + enqueue_job_required( + request_tag, + Target::X86_64UnknownLinuxGnu, + CodegenBackend::Llvm, + Profile::Opt, + 0u32, + BenchmarkJobKind::Rustc, + ) + .await?; + tx.conn() .update_benchmark_request_status(request_tag, BenchmarkRequestStatus::InProgress) .await?; From 1f72bebc1f72e55917ca42c3977713263cc2ee8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 13 Oct 2025 15:38:00 +0200 Subject: [PATCH 4/4] Add simple test --- database/src/tests/mod.rs | 4 ++++ site/src/job_queue/mod.rs | 25 ++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/database/src/tests/mod.rs b/database/src/tests/mod.rs index 9d88a60ce..b0afac3af 100644 --- a/database/src/tests/mod.rs +++ b/database/src/tests/mod.rs @@ -100,6 +100,10 @@ impl TestContext { self.connection.as_ref() } + pub fn db_mut(&mut self) -> &mut dyn Connection { + self.connection.as_mut() + } + async fn finish(self) { // Cleanup the test database // First, we need to stop using the database diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index 939a84e16..95690b02d 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -484,7 +484,7 @@ pub async fn create_queue_process( #[cfg(test)] mod tests { - use crate::job_queue::build_queue; + use crate::job_queue::{build_queue, process_benchmark_requests}; use chrono::Utc; use database::tests::run_postgres_test; use database::{ @@ -677,4 +677,27 @@ mod tests { }) .await; } + + #[tokio::test] + async fn insert_all_jobs() { + run_postgres_test(|mut ctx| async { + ctx.insert_master_request("bar", "baz", 1).await; + ctx.complete_request("bar").await; + ctx.insert_master_request("foo", "bar", 1).await; + + process_benchmark_requests(ctx.db_mut()).await?; + let jobs = ctx + .db() + .get_jobs_of_in_progress_benchmark_requests() + .await + .unwrap() + .remove("foo") + .unwrap(); + // runtime + rustc + 4 compile-time jobs + assert_eq!(jobs.len(), 6); + + Ok(ctx) + }) + .await; + } }