From 2b9832ed223575356f6c555293afc8d9c039650a Mon Sep 17 00:00:00 2001 From: Carbonhell Date: Sat, 15 Nov 2025 16:10:43 +0100 Subject: [PATCH 1/3] feat(cli): add cratesfyi command to queue rebuilds for specific broken nightly dates --- src/bin/cratesfyi.rs | 23 ++++- src/build_queue.rs | 234 +++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 4 +- 3 files changed, 259 insertions(+), 2 deletions(-) diff --git a/src/bin/cratesfyi.rs b/src/bin/cratesfyi.rs index 0eeefd036..5aea964e9 100644 --- a/src/bin/cratesfyi.rs +++ b/src/bin/cratesfyi.rs @@ -1,9 +1,10 @@ use anyhow::{Context as _, Result, anyhow}; +use chrono::NaiveDate; use clap::{Parser, Subcommand, ValueEnum}; use docs_rs::{ Config, Context, Index, PackageKind, RustwideBuilder, db::{self, CrateId, Overrides, ReleaseId, add_path_into_database, types::version::Version}, - start_background_metrics_webserver, start_web_server, + queue_rebuilds_faulty_rustdoc, start_background_metrics_webserver, start_web_server, utils::{ ConfigName, get_config, get_crate_pattern_and_priority, list_crate_priorities, queue_builder, remove_crate_priority, set_config, set_crate_priority, @@ -270,6 +271,17 @@ enum QueueSubcommand { #[arg(long, conflicts_with("reference"))] head: bool, }, + + /// Queue rebuilds for broken nightly versions of rustdoc + RebuildBrokenNightly { + /// Start date of nightly builds to rebuild (inclusive) + #[arg(name = "START", short = 's', long = "start")] + start_nightly_date: NaiveDate, + + /// End date of nightly builds to rebuild (exclusive, optional) + #[arg(name = "END", short = 'e', long = "end")] + end_nightly_date: Option, + }, } impl QueueSubcommand { @@ -312,6 +324,15 @@ impl QueueSubcommand { } Self::DefaultPriority { subcommand } => subcommand.handle_args(ctx)?, + + Self::RebuildBrokenNightly { start_nightly_date, end_nightly_date } => { + ctx.runtime.block_on(async move { + let mut conn = ctx.pool.get_async().await?; + let queued_rebuilds_amount = queue_rebuilds_faulty_rustdoc(&mut conn, &ctx.async_build_queue, &start_nightly_date, &end_nightly_date).await?; + println!("Queued {queued_rebuilds_amount} rebuilds for broken nightly versions of rustdoc"); + Ok::<(), anyhow::Error>(()) + })? + } } Ok(()) } diff --git a/src/build_queue.rs b/src/build_queue.rs index dafc84d1b..40c4cbb1a 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -10,6 +10,7 @@ use crate::{ utils::{ConfigName, get_config, get_crate_priority, report_error, retry, set_config}, }; use anyhow::Context as _; +use chrono::NaiveDate; use fn_error_context::context; use futures_util::{StreamExt, stream::TryStreamExt}; use sqlx::Connection as _; @@ -22,6 +23,8 @@ use tracing::{debug, error, info, instrument, warn}; /// collapsed in the UI. /// For normal build priorities we use smaller values. pub(crate) const REBUILD_PRIORITY: i32 = 20; +// TODO what value should we use here? +pub(crate) const BROKEN_RUSTDOC_REBUILD_PRIORITY: i32 = 30; #[derive(Debug, Clone, Eq, PartialEq, serde::Serialize)] pub(crate) struct QueuedCrate { @@ -726,9 +729,75 @@ pub async fn queue_rebuilds( Ok(()) } +/// Queue rebuilds for failed crates due to a faulty version of rustdoc +/// +/// It is assumed that the version of rustdoc matches the one of rustc, which is persisted in the DB. +/// The priority of the resulting rebuild requests will be lower than previously failed builds. +/// If a crate is already queued to be rebuilt, it will not be requeued. +/// Start date is inclusive, end date is exclusive. +#[instrument(skip_all)] +pub async fn queue_rebuilds_faulty_rustdoc( + conn: &mut sqlx::PgConnection, + build_queue: &AsyncBuildQueue, + start_nightly_date: &NaiveDate, + end_nightly_date: &Option, +) -> Result { + let end_nightly_date = + end_nightly_date.unwrap_or_else(|| start_nightly_date.succ_opt().unwrap()); + let mut results = sqlx::query!( + r#" +SELECT c.name, + r.version AS "version: Version" +FROM crates AS c + JOIN releases AS r + ON c.latest_version_id = r.id + AND r.rustdoc_status = TRUE + JOIN LATERAL ( + SELECT b.id, + b.build_status, + b.rustc_nightly_date, + COALESCE(b.build_finished, b.build_started) AS last_build_attempt + FROM builds AS b + WHERE b.rid = r.id + ORDER BY last_build_attempt DESC + LIMIT 1 + ) AS b ON b.build_status = 'failure' AND b.rustc_nightly_date >= $1 AND b.rustc_nightly_date < $2 + +"#, start_nightly_date, end_nightly_date + ) + .fetch(&mut *conn); + + let mut results_count = 0; + while let Some(row) = results.next().await { + let row = row?; + + if !build_queue + .has_build_queued(&row.name, &row.version) + .await? + { + results_count += 1; + info!( + "queueing rebuild for {} {} (priority {})...", + &row.name, &row.version, BROKEN_RUSTDOC_REBUILD_PRIORITY + ); + build_queue + .add_crate( + &row.name, + &row.version, + BROKEN_RUSTDOC_REBUILD_PRIORITY, + None, + ) + .await?; + } + } + + Ok(results_count) +} + #[cfg(test)] mod tests { use super::*; + use crate::db::types::BuildStatus; use crate::test::{FakeBuild, TestEnvironment, V1, V2}; use chrono::Utc; use std::time::Duration; @@ -767,6 +836,171 @@ mod tests { Ok(()) } + /// Verifies whether a rebuild is queued for a crate that previously failed with a nightly version of rustdoc. + #[tokio::test(flavor = "multi_thread")] + async fn test_rebuild_broken_rustdoc_specific_date_simple() -> Result<()> { + let env = TestEnvironment::with_config( + TestEnvironment::base_config() + .max_queued_rebuilds(Some(100)) + .build()?, + ) + .await?; + + for i in 1..5 { + let nightly_date = NaiveDate::from_ymd_opt(2020, 10, i).unwrap(); + env.fake_release() + .await + .name(&format!("foo{}", i)) + .version(V1) + .builds(vec![ + FakeBuild::default() + .rustc_version( + format!( + "rustc 1.84.0-nightly (e7c0d2750 {})", + nightly_date.format("%Y-%m-%d") + ) + .as_str(), + ) + .build_status(BuildStatus::Failure), + ]) + .create() + .await?; + } + + let build_queue = env.async_build_queue(); + assert!(build_queue.queued_crates().await?.is_empty()); + + let mut conn = env.async_db().async_conn().await; + queue_rebuilds_faulty_rustdoc( + &mut conn, + build_queue, + &NaiveDate::from_ymd_opt(2020, 10, 3).unwrap(), + &None, + ) + .await?; + + let queue = build_queue.queued_crates().await?; + assert_eq!(queue.len(), 1); + assert_eq!(queue[0].name, "foo3"); + assert_eq!(queue[0].version, V1); + assert_eq!(queue[0].priority, BROKEN_RUSTDOC_REBUILD_PRIORITY); + + Ok(()) + } + + /// Verified whether a rebuild is NOT queued since the latest build for the specific crate is marked as successful. + #[tokio::test(flavor = "multi_thread")] + async fn test_rebuild_broken_rustdoc_specific_date_skipped() -> Result<()> { + let env = TestEnvironment::with_config( + TestEnvironment::base_config() + .max_queued_rebuilds(Some(100)) + .build()?, + ) + .await?; + + env.fake_release() + .await + .name("foo") + .version(V1) + .builds(vec![ + FakeBuild::default() + .rustc_version( + format!( + "rustc 1.84.0-nightly (e7c0d2750 {})", + NaiveDate::from_ymd_opt(2020, 10, 1) + .unwrap() + .format("%Y-%m-%d") + ) + .as_str(), + ) + .build_status(BuildStatus::Failure), + FakeBuild::default() + .rustc_version( + format!( + "rustc 1.84.0-nightly (e7c0d2750 {})", + NaiveDate::from_ymd_opt(2020, 10, 1) + .unwrap() + .format("%Y-%m-%d") + ) + .as_str(), + ) + .build_status(BuildStatus::Success), + ]) + .create() + .await?; + + let build_queue = env.async_build_queue(); + assert!(build_queue.queued_crates().await?.is_empty()); + + let mut conn = env.async_db().async_conn().await; + queue_rebuilds_faulty_rustdoc( + &mut conn, + build_queue, + &NaiveDate::from_ymd_opt(2020, 10, 1).unwrap(), + &None, + ) + .await?; + + let queue = build_queue.queued_crates().await?; + assert_eq!(queue.len(), 0); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_rebuild_broken_rustdoc_date_range() -> Result<()> { + let env = TestEnvironment::with_config( + TestEnvironment::base_config() + .max_queued_rebuilds(Some(100)) + .build()?, + ) + .await?; + + for i in 1..6 { + let nightly_date = NaiveDate::from_ymd_opt(2020, 10, i).unwrap(); + env.fake_release() + .await + .name(&format!("foo{}", i)) + .version(V1) + .builds(vec![ + FakeBuild::default() + .rustc_version( + format!( + "rustc 1.84.0-nightly (e7c0d2750 {})", + nightly_date.format("%Y-%m-%d") + ) + .as_str(), + ) + .build_status(BuildStatus::Failure), + ]) + .create() + .await?; + } + + let build_queue = env.async_build_queue(); + assert!(build_queue.queued_crates().await?.is_empty()); + + let mut conn = env.async_db().async_conn().await; + queue_rebuilds_faulty_rustdoc( + &mut conn, + build_queue, + &NaiveDate::from_ymd_opt(2020, 10, 3).unwrap(), + &Some(NaiveDate::from_ymd_opt(2020, 10, 5)).unwrap(), + ) + .await?; + + let queue = build_queue.queued_crates().await?; + assert_eq!(queue.len(), 2); + assert_eq!(queue[0].name, "foo3"); + assert_eq!(queue[0].version, V1); + assert_eq!(queue[0].priority, BROKEN_RUSTDOC_REBUILD_PRIORITY); + assert_eq!(queue[1].name, "foo4"); + assert_eq!(queue[1].version, V1); + assert_eq!(queue[1].priority, BROKEN_RUSTDOC_REBUILD_PRIORITY); + + Ok(()) + } + #[tokio::test(flavor = "multi_thread")] async fn test_still_rebuild_when_full_with_failed() -> Result<()> { let env = TestEnvironment::with_config( diff --git a/src/lib.rs b/src/lib.rs index fc9d1423f..e45e42b16 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,9 @@ //! documentation of crates for the Rust Programming Language. #![allow(clippy::cognitive_complexity)] -pub use self::build_queue::{AsyncBuildQueue, BuildQueue, queue_rebuilds}; +pub use self::build_queue::{ + AsyncBuildQueue, BuildQueue, queue_rebuilds, queue_rebuilds_faulty_rustdoc, +}; pub use self::config::Config; pub use self::context::Context; pub use self::docbuilder::PackageKind; From c0a2af5088dcd24997e766a694d7fc2a57b0f6a7 Mon Sep 17 00:00:00 2001 From: Carbonhell Date: Sat, 15 Nov 2025 16:35:43 +0100 Subject: [PATCH 2/3] chore(sqlx): add updated sqlx prepare artifact --- ...067c225de3d323f74d36512c579a6896c68b6.json | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .sqlx/query-64031f2e97958452a6d6da840fd067c225de3d323f74d36512c579a6896c68b6.json diff --git a/.sqlx/query-64031f2e97958452a6d6da840fd067c225de3d323f74d36512c579a6896c68b6.json b/.sqlx/query-64031f2e97958452a6d6da840fd067c225de3d323f74d36512c579a6896c68b6.json new file mode 100644 index 000000000..730433aea --- /dev/null +++ b/.sqlx/query-64031f2e97958452a6d6da840fd067c225de3d323f74d36512c579a6896c68b6.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\nSELECT c.name,\n r.version AS \"version: Version\"\nFROM crates AS c\n JOIN releases AS r\n ON c.latest_version_id = r.id\n AND r.rustdoc_status = TRUE\n JOIN LATERAL (\n SELECT b.id,\n b.build_status,\n b.rustc_nightly_date,\n COALESCE(b.build_finished, b.build_started) AS last_build_attempt\n FROM builds AS b\n WHERE b.rid = r.id\n ORDER BY last_build_attempt DESC\n LIMIT 1\n ) AS b ON b.build_status = 'failure' AND b.rustc_nightly_date >= $1 AND b.rustc_nightly_date < $2\n\n", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "version: Version", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Date", + "Date" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "64031f2e97958452a6d6da840fd067c225de3d323f74d36512c579a6896c68b6" +} From 40fbec130c29108c1d0290006d8b3420a538119b Mon Sep 17 00:00:00 2001 From: Carbonhell Date: Sat, 15 Nov 2025 16:40:05 +0100 Subject: [PATCH 3/3] chore(tests): remove unneeded Some().unwrap() --- src/build_queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build_queue.rs b/src/build_queue.rs index 40c4cbb1a..2649ea871 100644 --- a/src/build_queue.rs +++ b/src/build_queue.rs @@ -985,7 +985,7 @@ mod tests { &mut conn, build_queue, &NaiveDate::from_ymd_opt(2020, 10, 3).unwrap(), - &Some(NaiveDate::from_ymd_opt(2020, 10, 5)).unwrap(), + &NaiveDate::from_ymd_opt(2020, 10, 5), ) .await?;