From 8ff0df6b9d9acdbd3a6a27238b7c3f5184b06755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 20 Oct 2025 15:37:52 +0200 Subject: [PATCH 1/3] Remove cutoff dates and instead enable enabling the job queue insertion with an environment variable --- site/src/job_queue/mod.rs | 84 ++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index ec294e647..b3363ca26 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -21,6 +21,14 @@ pub fn is_job_queue_enabled() -> bool { .unwrap_or(true) } +/// Should master and release artifacts be automatically inserted into the job queue? +pub fn is_job_queue_insertion_enabled() -> bool { + std::env::var("INSERT_REQUESTS_INTO_JOB_QUEUE") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(false) +} + /// rust-lang/rust PR that will be used for testing the job queue. const TEST_PR_FOR_JOB_QUEUE: u32 = 147039; @@ -33,7 +41,7 @@ pub fn should_use_job_queue(pr: u32) -> bool { /// Returns `true` if at least one benchmark request was inserted. async fn create_benchmark_request_master_commits( ctxt: &SiteCtxt, - _conn: &dyn database::pool::Connection, + conn: &dyn database::pool::Connection, index: &BenchmarkRequestIndex, ) -> anyhow::Result { let now = Utc::now(); @@ -45,29 +53,23 @@ async fn create_benchmark_request_master_commits( .iter() .filter(|c| now.signed_duration_since(c.time) < chrono::Duration::days(29)); - // TODO; delete at some point in the future - let cutoff: chrono::DateTime = chrono::DateTime::from_str("2025-08-27T00:00:00.000Z")?; - - let inserted = false; + let mut inserted = false; for master_commit in master_commits { - // We don't want to add masses of obsolete data - if master_commit.time >= cutoff && !index.contains_tag(&master_commit.sha) { - // let pr = master_commit.pr.unwrap_or(0); - // let benchmark = BenchmarkRequest::create_master( - // &master_commit.sha, - // &master_commit.parent_sha, - // pr, - // master_commit.time, - // ); - // log::info!("Inserting master benchmark request {benchmark:?}"); - - // Do not create benchmark requests on production, to allow running in parallel with - // the old system. - // if let Err(error) = conn.insert_benchmark_request(&benchmark).await { - // log::error!("Failed to insert master benchmark request: {error:?}"); - // } else { - // inserted = true; - // } + if !index.contains_tag(&master_commit.sha) { + let pr = master_commit.pr.unwrap_or(0); + let benchmark = BenchmarkRequest::create_master( + &master_commit.sha, + &master_commit.parent_sha, + pr, + master_commit.time, + ); + log::info!("Inserting master benchmark request {benchmark:?}"); + + if let Err(error) = conn.insert_benchmark_request(&benchmark).await { + log::error!("Failed to insert master benchmark request: {error:?}"); + } else { + inserted = true; + } } } Ok(inserted) @@ -77,15 +79,13 @@ async fn create_benchmark_request_master_commits( /// already in the database /// Returns `true` if at least one benchmark request was inserted. async fn create_benchmark_request_releases( - _conn: &dyn database::pool::Connection, + conn: &dyn database::pool::Connection, index: &BenchmarkRequestIndex, ) -> anyhow::Result { let releases: String = reqwest::get("https://static.rust-lang.org/manifests.txt") .await? .text() .await?; - // TODO; delete at some point in the future - let cutoff: chrono::DateTime = chrono::DateTime::from_str("2025-08-27T00:00:00.000Z")?; let releases = releases .lines() @@ -93,19 +93,17 @@ async fn create_benchmark_request_releases( .filter_map(parse_release_string) .take(20); - let inserted = false; + let mut inserted = false; for (name, commit_date) in releases { - if commit_date >= cutoff && !index.contains_tag(&name) { - // let release_request = BenchmarkRequest::create_release(&name, commit_date); - // log::info!("Inserting release benchmark request {release_request:?}"); - - // Do not create benchmark requests on production, to allow running in parallel with - // the old system. - // if let Err(error) = conn.insert_benchmark_request(&release_request).await { - // log::error!("Failed to insert release benchmark request: {error}"); - // } else { - // inserted = true; - // } + if !index.contains_tag(&name) { + let release_request = BenchmarkRequest::create_release(&name, commit_date); + log::info!("Inserting release benchmark request {release_request:?}"); + + if let Err(error) = conn.insert_benchmark_request(&release_request).await { + log::error!("Failed to insert release benchmark request: {error}"); + } else { + inserted = true; + } } } Ok(inserted) @@ -388,11 +386,17 @@ async fn perform_queue_tick(ctxt: &SiteCtxt) -> anyhow::Result<()> { let index = ctxt.known_benchmark_requests.load(); + let insertion_enabled = is_job_queue_insertion_enabled(); + let mut requests_inserted = false; // Put the master commits into the `benchmark_requests` queue - requests_inserted |= create_benchmark_request_master_commits(ctxt, &*conn, &index).await?; + if insertion_enabled { + requests_inserted |= create_benchmark_request_master_commits(ctxt, &*conn, &index).await?; + } // Put the releases into the `benchmark_requests` queue - requests_inserted |= create_benchmark_request_releases(&*conn, &index).await?; + if insertion_enabled { + requests_inserted |= create_benchmark_request_releases(&*conn, &index).await?; + } // Enqueue waiting requests and try to complete in-progress ones let completed_reqs = process_benchmark_requests(&mut *conn).await?; From f43227246fb2d76381b224f996e71c03a8472eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Mon, 20 Oct 2025 15:40:04 +0200 Subject: [PATCH 2/3] Do not abort the job queue if master or release artifacts cannot be created We do not necessarily have to enqueue these artifacts immediately, if their web resources are not working, we should still be able to continue with operating the job queue. --- site/src/job_queue/mod.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index b3363ca26..f347d8077 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -391,11 +391,25 @@ async fn perform_queue_tick(ctxt: &SiteCtxt) -> anyhow::Result<()> { let mut requests_inserted = false; // Put the master commits into the `benchmark_requests` queue if insertion_enabled { - requests_inserted |= create_benchmark_request_master_commits(ctxt, &*conn, &index).await?; + match create_benchmark_request_master_commits(ctxt, &*conn, &index).await { + Ok(inserted) => requests_inserted |= inserted, + Err(error) => { + log::error!( + "Could not insert master benchmark requests into the database: {error:?}" + ); + } + } } // Put the releases into the `benchmark_requests` queue if insertion_enabled { - requests_inserted |= create_benchmark_request_releases(&*conn, &index).await?; + match create_benchmark_request_releases(&*conn, &index).await { + Ok(inserted) => requests_inserted |= inserted, + Err(error) => { + log::error!( + "Could not insert release benchmark requests into the database: {error:?}" + ); + } + } } // Enqueue waiting requests and try to complete in-progress ones let completed_reqs = process_benchmark_requests(&mut *conn).await?; From 0d12f899f0f37e77e431a2762e010c77b0a0f2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= Date: Tue, 21 Oct 2025 09:38:34 +0200 Subject: [PATCH 3/3] Fix Clippy --- site/src/comparison.rs | 4 ++-- site/src/job_queue/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/site/src/comparison.rs b/site/src/comparison.rs index 010f2c7c0..b34f2ad21 100644 --- a/site/src/comparison.rs +++ b/site/src/comparison.rs @@ -1134,7 +1134,7 @@ impl HistoricalData { fn quartiles(&self) -> (f64, f64) { let pcs = self.percent_changes(); fn median(data: &[f64]) -> f64 { - if data.len() % 2 == 0 { + if data.len().is_multiple_of(2) { (data[(data.len() - 1) / 2] + data[data.len() / 2]) / 2.0 } else { data[data.len() / 2] @@ -1144,7 +1144,7 @@ impl HistoricalData { let len = pcs.len(); let (h1_end, h2_begin) = if len <= 2 { (0, std::cmp::min(len, 1)) - } else if len % 2 == 0 { + } else if len.is_multiple_of(2) { (len / 2 - 2, len / 2 + 1) } else { (len / 2 - 1, len / 2 + 1) diff --git a/site/src/job_queue/mod.rs b/site/src/job_queue/mod.rs index f347d8077..63eefe466 100644 --- a/site/src/job_queue/mod.rs +++ b/site/src/job_queue/mod.rs @@ -11,7 +11,7 @@ use database::{ Target, }; use parking_lot::RwLock; -use std::{str::FromStr, sync::Arc}; +use std::sync::Arc; use tokio::time::{self, Duration, MissedTickBehavior}; pub fn is_job_queue_enabled() -> bool {