From 57937da16bdb9b05063501e5cfcd6a34577b13ee Mon Sep 17 00:00:00 2001 From: Arnaud Ferraris Date: Thu, 2 Nov 2023 12:55:38 +0100 Subject: [PATCH 1/3] db: relax version checking When calling `search_generic()`, the `version` parameter is taken from the output of `cargo metadata`, which always return the latest version available on crates.io. Due to ba11751 this implies the crate will be considered missing from Debian unless the packaged version is the latest one. This commit adds a fallback version check in case the initial check fails, by trimming the version to a supposedly compatible semver version. Fixes #31 --- src/db.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/db.rs b/src/db.rs index 8842317..26deca7 100644 --- a/src/db.rs +++ b/src/db.rs @@ -130,17 +130,26 @@ impl Connection { version: &Version, ) -> Result { let package = package.replace('_', "-"); - let semver_package = if version.major == 0 { - format!("rust-{package}-{}.{}", version.major, version.minor) + let semver_version = if version.major == 0 { + if version.minor == 0 { + format!("{}.{}.{}", version.major, version.minor, version.patch) + } else { + format!("{}.{}", version.major, version.minor) + } } else { - format!("rust-{package}-{}", version.major) + format!("{}", version.major) }; - let rows = self - .sock - .query(query, &[&format!("rust-{package}"), &semver_package])?; + let rows = self.sock.query( + query, + &[ + &format!("rust-{package}"), + &format!("rust-{package}-{}", semver_version), + ], + )?; let version = version.to_string(); let version = VersionReq::parse(&version)?; + let semver_version = VersionReq::parse(&semver_version)?; for row in &rows { let debversion: String = row.get(0); @@ -151,7 +160,7 @@ impl Connection { // println!("{:?} ({:?}) => {:?}", debversion, version, is_compatible(debversion, version)?); - if is_compatible(debversion, &version)? { + if is_compatible(debversion, &version)? || is_compatible(debversion, &semver_version)? { return Ok(true); } } From 18897d72ad15d27fa3d6a1beaf76b084a3584012 Mon Sep 17 00:00:00 2001 From: Arnaud Ferraris Date: Thu, 2 Nov 2023 13:30:43 +0100 Subject: [PATCH 2/3] Implement "outdated" check and introduce "compatible" version Package search only returns a boolean depending on whether a package with a correct version exists in Debian (or NEW). This isn't sufficient to handle the cases where the package exists, but is so outdated that the version in Debian is likely incompatible with the crate we're inspecting. Moreover, this doesn't differentiate between exact version matches and semver'd matches (e.g. when the Debian version is older but has the same major -- and, if needed, minor -- as the version found on crates.io). In order to deal with the latter case, this change introduces the notion of "compatible" version: it represents crates already packaged in Debian but not on the latest version, although it *should* still be compatible with the requirements. The "compatible" and "outdated" information are then propagated, as well as the Debian package version, through a new `db::PkgInfo` structure used as the return type for the `db::search*` functions. That way, this additional information can easily be processed by other modules and ultimately be reported to the user. --- src/db.rs | 81 +++++++++++++++++++++++++++++++++-------------- src/debian.rs | 24 +++++++++++--- src/format/mod.rs | 24 ++++++++++++-- 3 files changed, 98 insertions(+), 31 deletions(-) diff --git a/src/db.rs b/src/db.rs index 26deca7..a2d1d7b 100644 --- a/src/db.rs +++ b/src/db.rs @@ -9,13 +9,26 @@ use std::time::{Duration, SystemTime}; const POSTGRES: &str = "postgresql://udd-mirror:udd-mirror@udd-mirror.debian.net/udd"; const CACHE_EXPIRE: Duration = Duration::from_secs(90 * 60); +#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] +pub enum PkgStatus { + NotFound, + Outdated, + Compatible, + Found, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct PkgInfo { + pub status: PkgStatus, + pub version: String, +} + #[derive(Debug, Serialize, Deserialize)] pub struct CacheEntry { pub from: SystemTime, - pub found: bool, + pub info: PkgInfo, } -// TODO: also use this for outdated check(?) fn is_compatible(debversion: &str, crateversion: &VersionReq) -> Result { let debversion = debversion.replace('~', "-"); let debversion = Version::parse(&debversion)?; @@ -56,20 +69,28 @@ impl Connection { target: &str, package: &str, version: &Version, - ) -> Result, Error> { + ) -> Result, Error> { let path = self.cache_path(target, package, version); if !path.exists() { return Ok(None); } - let buf = fs::read(path)?; - let cache: CacheEntry = serde_json::from_slice(&buf)?; + let buf = fs::read(&path)?; + // If the cache entry can't be deserialized, it's probably using an old + // entry format, so let's discard it + let cache: CacheEntry = match serde_json::from_slice(&buf) { + Ok(e) => e, + _ => { + fs::remove_file(path)?; + return Ok(None); + } + }; if SystemTime::now().duration_since(cache.from)? > CACHE_EXPIRE { Ok(None) } else { - Ok(Some(cache.found)) + Ok(Some(cache.info)) } } @@ -78,49 +99,49 @@ impl Connection { target: &str, package: &str, version: &Version, - found: bool, + info: &PkgInfo, ) -> Result<(), Error> { let cache = CacheEntry { from: SystemTime::now(), - found, + info: info.clone(), }; let buf = serde_json::to_vec(&cache)?; fs::write(self.cache_path(target, package, version), buf)?; Ok(()) } - pub fn search(&mut self, package: &str, version: &Version) -> Result { - if let Some(found) = self.check_cache("sid", package, version)? { - return Ok(found); + pub fn search(&mut self, package: &str, version: &Version) -> Result { + if let Some(info) = self.check_cache("sid", package, version)? { + return Ok(info); } // config.shell().status("Querying", format!("sid: {}", package))?; info!("Querying -> sid: {}", package); - let found = self.search_generic( + let info = self.search_generic( "SELECT version::text FROM sources WHERE source in ($1, $2) AND release='sid';", package, version, )?; - self.write_cache("sid", package, version, found)?; - Ok(found) + self.write_cache("sid", package, version, &info)?; + Ok(info) } - pub fn search_new(&mut self, package: &str, version: &Version) -> Result { - if let Some(found) = self.check_cache("new", package, version)? { - return Ok(found); + pub fn search_new(&mut self, package: &str, version: &Version) -> Result { + if let Some(info) = self.check_cache("new", package, version)? { + return Ok(info); } // config.shell().status("Querying", format!("new: {}", package))?; info!("Querying -> new: {}", package); - let found = self.search_generic( + let info = self.search_generic( "SELECT version::text FROM new_sources WHERE source in ($1, $2);", package, version, )?; - self.write_cache("new", package, version, found)?; - Ok(found) + self.write_cache("new", package, version, &info)?; + Ok(info) } pub fn search_generic( @@ -128,7 +149,11 @@ impl Connection { query: &str, package: &str, version: &Version, - ) -> Result { + ) -> Result { + let mut info = PkgInfo { + status: PkgStatus::NotFound, + version: String::new(), + }; let package = package.replace('_', "-"); let semver_version = if version.major == 0 { if version.minor == 0 { @@ -160,12 +185,20 @@ impl Connection { // println!("{:?} ({:?}) => {:?}", debversion, version, is_compatible(debversion, version)?); - if is_compatible(debversion, &version)? || is_compatible(debversion, &semver_version)? { - return Ok(true); + if is_compatible(debversion, &version)? { + info.version = debversion.to_string(); + info.status = PkgStatus::Found; + return Ok(info); + } else if is_compatible(debversion, &semver_version)? { + info.version = debversion.to_string(); + info.status = PkgStatus::Compatible; + } else if info.status == PkgStatus::NotFound { + info.version = debversion.to_string(); + info.status = PkgStatus::Outdated; } } - Ok(false) + Ok(info) } } diff --git a/src/debian.rs b/src/debian.rs index 53f4805..46b8703 100644 --- a/src/debian.rs +++ b/src/debian.rs @@ -1,4 +1,4 @@ -use crate::db::Connection; +use crate::db::{Connection, PkgStatus}; use crate::errors::*; use crate::graph::Graph; use cargo_metadata::{Package, PackageId, Source}; @@ -52,6 +52,8 @@ pub struct DebianInfo { pub in_unstable: bool, pub in_new: bool, pub outdated: bool, + pub compatible: bool, + pub version: String, } fn run_task(db: &mut Connection, pkg: Pkg) -> Result { @@ -59,12 +61,26 @@ fn run_task(db: &mut Connection, pkg: Pkg) -> Result { in_unstable: false, in_new: false, outdated: false, + compatible: false, + version: String::new(), }; - if db.search(&pkg.name, &pkg.version).unwrap() { + let mut info = db.search(&pkg.name, &pkg.version).unwrap(); + if info.status == PkgStatus::NotFound { + info = db.search_new(&pkg.name, &pkg.version).unwrap(); + if info.status != PkgStatus::NotFound { + deb.in_new = true; + deb.version = info.version; + } + } else { deb.in_unstable = true; - } else if db.search_new(&pkg.name, &pkg.version).unwrap() { - deb.in_new = true; + deb.version = info.version; + } + + match info.status { + PkgStatus::Outdated => deb.outdated = true, + PkgStatus::Compatible => deb.compatible = true, + _ => (), } Ok(deb) diff --git a/src/format/mod.rs b/src/format/mod.rs index b4d212a..103d5ee 100644 --- a/src/format/mod.rs +++ b/src/format/mod.rs @@ -58,11 +58,29 @@ impl<'a> fmt::Display for Display<'a> { let pkg = format!("{} v{}", self.package.name, self.package.version); if let Some(deb) = &self.package.debinfo { if deb.in_unstable { - write!(fmt, "{} (in debian)", pkg.green())?; + if deb.compatible { + write!( + fmt, + "{} ({} in debian)", + pkg.green(), + deb.version.yellow() + )?; + } else { + write!(fmt, "{} (in debian)", pkg.green())?; + } } else if deb.in_new { - write!(fmt, "{} (in debian NEW queue)", pkg.blue())?; + if deb.compatible { + write!( + fmt, + "{} ({} in debian NEW queue)", + pkg.blue(), + deb.version.yellow() + )?; + } else { + write!(fmt, "{} (in debian NEW queue)", pkg.blue())?; + } } else if deb.outdated { - write!(fmt, "{} (outdated)", pkg.yellow())?; + write!(fmt, "{} (outdated, {})", pkg.red(), deb.version)?; } else { write!(fmt, "{pkg}")?; } From 122912d9991484796254a5d7d4dfc2071599d172 Mon Sep 17 00:00:00 2001 From: Arnaud Ferraris Date: Mon, 6 Nov 2023 13:29:46 +0100 Subject: [PATCH 3/3] db: add test for new version check logic This new test searches for `serde` in Debian Bullseye, which is `oldstable` now and thus won't be updated anymore. It requests various versions to ensure the crate is reported as "found", "outdated" or "compatible" appropriately. It also searches for the non-existing `notacrate` to ensure it properly reports missing packages. Finally, this test (which requires network access) is marked as ignored so it isn't run by default, avoiding issues with Debian's build process. The workflow has been updated to ensure this test is run in CI. --- .github/workflows/rust.yml | 2 +- src/db.rs | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 35a5a98..6ae9d11 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -19,7 +19,7 @@ jobs: - name: Build run: cargo build --verbose - name: Run tests - run: cargo test --verbose + run: cargo test --verbose -- --include-ignored - name: Run debstatus on itself run: cargo run -- debstatus diff --git a/src/db.rs b/src/db.rs index a2d1d7b..b2a8d37 100644 --- a/src/db.rs +++ b/src/db.rs @@ -204,8 +204,8 @@ impl Connection { #[cfg(test)] mod tests { - use crate::db::is_compatible; - use semver::VersionReq; + use crate::db::{is_compatible, Connection, PkgStatus}; + use semver::{Version, VersionReq}; #[test] fn is_compatible_with_tilde() { @@ -222,4 +222,30 @@ mod tests { assert!(!is_compatible("0.1.0", &VersionReq::parse("0.1.1").unwrap()).unwrap()); assert!(is_compatible("1.1.0", &VersionReq::parse("1").unwrap()).unwrap()); } + + #[test] + #[ignore] + fn check_version_reqs() { + let mut db = Connection::new().unwrap(); + // Debian bullseye has rust-serde v1.0.106 and shouldn't be updated anymore + let query = + "SELECT version::text FROM sources WHERE source in ($1, $2) AND release='bullseye';"; + let info = db + .search_generic(query, "serde", &Version::parse("1.0.100").unwrap()) + .unwrap(); + assert_eq!(info.status, PkgStatus::Found); + assert_eq!(info.version, "1.0.106"); + let info = db + .search_generic(query, "serde", &Version::parse("1.0.150").unwrap()) + .unwrap(); + assert_eq!(info.status, PkgStatus::Compatible); + let info = db + .search_generic(query, "serde", &Version::parse("2.0.0").unwrap()) + .unwrap(); + assert_eq!(info.status, PkgStatus::Outdated); + let info = db + .search_generic(query, "notacrate", &Version::parse("1.0.0").unwrap()) + .unwrap(); + assert_eq!(info.status, PkgStatus::NotFound); + } }