From bde0894ef2862e0e07899979c073d91aff1d6353 Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Wed, 20 May 2026 21:12:30 -0700 Subject: [PATCH 1/2] feat: pin downloaded binaries with sha256 verification Every binary the runner downloads at install time (the patched valgrind .deb, the memtrack/exec-harness/mongo-tracer installer scripts) is now SHA-256-pinned. URLs and expected hashes live together in a new `PinnedBinary` enum, and the download helper rejects the install if the bytes don't match. Bumping a pinned version requires updating both the version constant and the matching hash; CONTRIBUTING.md documents the regeneration workflow. Co-Authored-By: Claude --- CONTRIBUTING.md | 29 +++++++---- src/binary_installer/mod.rs | 27 ++++------ src/binary_pins.rs | 76 ++++++++++++++++++++++++++++ src/cli/run/helpers/download_file.rs | 26 +++++++++- src/cli/run/helpers/mod.rs | 2 +- src/executor/memory/setup.rs | 11 ++-- src/executor/orchestrator.rs | 13 ++--- src/executor/valgrind/setup.rs | 69 ++++++++++++------------- src/instruments/mongo_tracer.rs | 15 ++---- src/lib.rs | 6 +-- 10 files changed, 179 insertions(+), 95 deletions(-) create mode 100644 src/binary_pins.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e5c89155..7e7de2e0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,19 +35,24 @@ cargo release -p exec-harness --execute beta After releasing `memtrack` or `exec-harness`, you **must** update the version references in the runner code: -1. **For memtrack**: Update `MEMTRACK_CODSPEED_VERSION` in `src/executor/memory/executor.rs`: +1. **For memtrack**: Update `MEMTRACK_VERSION` in `src/binary_pins.rs` and the matching SHA-256 in `PinnedBinary::sha256` (see [Pinned binary hashes](#pinned-binary-hashes) below). - ```rust - const MEMTRACK_CODSPEED_VERSION: &str = "X.Y.Z"; // Update to new version - ``` - -2. **For exec-harness**: Update `EXEC_HARNESS_VERSION` in `src/executor/orchestrator.rs`: - ```rust - const EXEC_HARNESS_VERSION: &str = "X.Y.Z"; // Update to new version - ``` +2. **For exec-harness**: Update `EXEC_HARNESS_VERSION` in `src/binary_pins.rs` and the matching SHA-256 in `PinnedBinary::sha256`. These constants are used by the runner to download and install the correct versions of the binaries from GitHub releases. +### Pinned binary hashes + +Every binary the runner downloads at install time (the patched valgrind `.deb`, the memtrack installer, the exec-harness installer, the mongo-tracer installer) is SHA-256-pinned. URLs and hashes live together in `PinnedBinary` in `src/binary_pins.rs`. + +When you bump a pinned version, regenerate the hash for each affected URL and update the matching match arm in `PinnedBinary::sha256`: + +```bash +curl -sL '' | sha256sum +``` + +For valgrind, that is one hash per supported `(distro_version, arch)` combination. + ### Releasing the Main Runner The main runner (`codspeed-runner`) should be released after ensuring all dependency versions are correct. @@ -56,8 +61,10 @@ The main runner (`codspeed-runner`) should be released after ensuring all depend **Verify binary version references**: Check that version constants in the runner code match the released versions: -- `MEMTRACK_CODSPEED_VERSION` in `src/executor/memory/executor.rs` -- `EXEC_HARNESS_VERSION` in `src/executor/orchestrator.rs` +- `MEMTRACK_VERSION` in `src/binary_pins.rs` +- `EXEC_HARNESS_VERSION` in `src/binary_pins.rs` + +Also confirm the SHA-256 entries in `PinnedBinary::sha256` (in `src/binary_pins.rs`) match the released artifacts. #### Release Command diff --git a/src/binary_installer/mod.rs b/src/binary_installer/mod.rs index c4f488f0..d8bdb75b 100644 --- a/src/binary_installer/mod.rs +++ b/src/binary_installer/mod.rs @@ -1,29 +1,26 @@ -use crate::cli::run::helpers::download_file; +use crate::binary_pins::PinnedBinary; +use crate::cli::run::helpers::download_pinned_file; use crate::prelude::*; use semver::Version; use std::process::Command; use tempfile::NamedTempFile; -use url::Url; mod versions; -/// Ensure a binary is installed, or install it from a runner's GitHub release using the installer script. +/// Ensure a binary is installed, or install it from a `PinnedBinary` installer script. /// /// This function checks if the binary is already installed with the correct version. -/// If not, it downloads and executes an installer script from the CodSpeed runner repository. +/// If not, it downloads and executes the pinned installer script. /// /// # Arguments /// * `binary_name` - The binary command name (e.g., "codspeed-memtrack", "codspeed-exec-harness") /// * `version` - The version to install (e.g., "4.4.2-alpha.2") -/// * `get_installer_url` - A closure that returns the URL to download the installer script. -pub async fn ensure_binary_installed( +/// * `installer` - The `PinnedBinary` installer to download. +pub async fn ensure_binary_installed( binary_name: &str, version: &str, - get_installer_url: F, -) -> Result<()> -where - F: FnOnce() -> String, -{ + installer: PinnedBinary, +) -> Result<()> { if is_command_installed( binary_name, Version::parse(version).context("Invalid version format")?, @@ -32,13 +29,11 @@ where return Ok(()); } - let installer_url = Url::parse(&get_installer_url()).context("Invalid installer URL")?; + debug!("Downloading installer for {binary_name}"); - debug!("Downloading installer from: {installer_url}"); - - // Download the installer script to a temporary file + // Download the installer script to a temporary file (with sha256 verification) let temp_file = NamedTempFile::new().context("Failed to create temporary file")?; - download_file(&installer_url, temp_file.path()).await?; + download_pinned_file(installer, temp_file.path()).await?; // Execute the installer script let output = Command::new("sh") diff --git a/src/binary_pins.rs b/src/binary_pins.rs new file mode 100644 index 00000000..b6ac6353 --- /dev/null +++ b/src/binary_pins.rs @@ -0,0 +1,76 @@ +// Pinned downloads. Each `PinnedBinary` variant carries the URL and the +// expected SHA-256 of the file at that URL — bumping a version requires +// updating both. See CONTRIBUTING.md for the regeneration workflow. + +pub const VALGRIND_DEB_VERSION: &str = "3.26.0-0codspeed0"; +pub const MEMTRACK_VERSION: &str = "1.2.3"; +pub const EXEC_HARNESS_VERSION: &str = "1.3.0"; +pub const MONGODB_TRACER_VERSION: &str = "cs-mongo-tracer-v0.2.0"; + +/// A binary the runner downloads at install time. The download helper looks +/// up the URL and SHA-256 via `url()` and `sha256()` and rejects the install +/// if the bytes don't match. +#[derive(Debug, Clone, Copy)] +pub enum PinnedBinary { + ValgrindDeb { + distro_version: &'static str, + arch: &'static str, + }, + MemtrackInstaller, + ExecHarnessInstaller, + MongoTracerInstaller, +} + +impl PinnedBinary { + pub fn url(&self) -> String { + match self { + PinnedBinary::ValgrindDeb { + distro_version, + arch, + } => format!( + "https://github.com/CodSpeedHQ/valgrind-codspeed/releases/download/{VALGRIND_DEB_VERSION}/valgrind_{VALGRIND_DEB_VERSION}_ubuntu-{distro_version}_{arch}.deb" + ), + PinnedBinary::MemtrackInstaller => format!( + "https://github.com/CodSpeedHQ/codspeed/releases/download/memtrack-v{MEMTRACK_VERSION}/memtrack-installer.sh" + ), + PinnedBinary::ExecHarnessInstaller => format!( + "https://github.com/CodSpeedHQ/codspeed/releases/download/exec-harness-v{EXEC_HARNESS_VERSION}/exec-harness-installer.sh" + ), + PinnedBinary::MongoTracerInstaller => format!( + "https://codspeed-public-assets.s3.eu-west-1.amazonaws.com/mongo-tracer/{MONGODB_TRACER_VERSION}/cs-mongo-tracer-installer.sh" + ), + } + } + + pub fn sha256(&self) -> &'static str { + match self { + PinnedBinary::ValgrindDeb { + distro_version, + arch, + } => match (*distro_version, *arch) { + ("22.04", "amd64") => { + "e0743f01668d664a97d85903057da7557dbf2dbbf0ceeb88e6b67ae9fb2f392f" + } + ("24.04", "amd64") => { + "2f02fd8e9377168310258ea03b356f4fa4beda3557e50da85681a3df889a886a" + } + ("22.04", "arm64") => { + "f24fc0676b8e0fd16de8efb8ab2a1dec0083304469883c03e553fb881dd5df29" + } + ("24.04", "arm64") => { + "32a48910da4b094192ef5c83e6526712fbf03b3d76a12e84423cbf6cc9ecbadf" + } + (d, a) => unreachable!("unknown valgrind target {d}/{a}"), + }, + PinnedBinary::MemtrackInstaller => { + "67f30ebe17d5da4246b51d8663394026385d95203ff09e81289772159e969603" + } + PinnedBinary::ExecHarnessInstaller => { + "75cbff4fdaefe98927d24fff43fd600c621eb1263b0c40b0fd32c68fa6d88ebd" + } + PinnedBinary::MongoTracerInstaller => { + "685f1d540cb24c2aa6f447991958339c6b70ec7664df2dba2713b8b3d77687e7" + } + } + } +} diff --git a/src/cli/run/helpers/download_file.rs b/src/cli/run/helpers/download_file.rs index ce9df1d8..0c0fb931 100644 --- a/src/cli/run/helpers/download_file.rs +++ b/src/cli/run/helpers/download_file.rs @@ -1,9 +1,10 @@ +use crate::binary_pins::PinnedBinary; use crate::{prelude::*, request_client::REQUEST_CLIENT}; use std::path::Path; use url::Url; -pub async fn download_file(url: &Url, path: &Path) -> Result<()> { +async fn download_file(url: &Url, path: &Path) -> Result<()> { debug!("Downloading file: {url}"); let response = REQUEST_CLIENT .get(url.clone()) @@ -23,3 +24,26 @@ pub async fn download_file(url: &Url, path: &Path) -> Result<()> { .map_err(|e| anyhow!("Failed to write to file: {}, {}", path.display(), e))?; Ok(()) } + +/// Download a `PinnedBinary` and verify its bytes against the SHA-256 +/// declared by `PinnedBinary::sha256`. On mismatch the partial file is +/// removed and an error is returned. +pub async fn download_pinned_file(binary: PinnedBinary, path: &Path) -> Result<()> { + let url_str = binary.url(); + let url = Url::parse(&url_str).context("failed to parse pinned URL")?; + download_file(&url, path).await?; + + let actual = sha256::try_digest(path) + .with_context(|| format!("failed to compute sha256 of {}", path.display()))?; + let expected = binary.sha256(); + + if actual != expected { + let _ = std::fs::remove_file(path); + bail!( + "Hash mismatch for {url_str}: expected {expected}, got {actual}. The downloaded file has been deleted." + ); + } + + debug!("Verified sha256 of {url_str}"); + Ok(()) +} diff --git a/src/cli/run/helpers/mod.rs b/src/cli/run/helpers/mod.rs index 98af84a2..09c05191 100644 --- a/src/cli/run/helpers/mod.rs +++ b/src/cli/run/helpers/mod.rs @@ -5,7 +5,7 @@ mod format_memory; mod get_env_var; mod parse_git_remote; -pub(crate) use download_file::download_file; +pub(crate) use download_file::download_pinned_file; pub(crate) use find_repository_root::find_repository_root; pub(crate) use format_duration::format_duration; pub(crate) use format_memory::format_memory; diff --git a/src/executor/memory/setup.rs b/src/executor/memory/setup.rs index 3db3aada..bab8b6d9 100644 --- a/src/executor/memory/setup.rs +++ b/src/executor/memory/setup.rs @@ -1,10 +1,11 @@ use crate::binary_installer::ensure_binary_installed; +use crate::binary_pins::{self, PinnedBinary}; use crate::executor::{ToolInstallStatus, ToolStatus}; use crate::prelude::*; use std::process::Command; pub const MEMTRACK_COMMAND: &str = "codspeed-memtrack"; -pub const MEMTRACK_CODSPEED_VERSION: &str = "1.2.3"; +pub const MEMTRACK_CODSPEED_VERSION: &str = binary_pins::MEMTRACK_VERSION; pub fn get_memtrack_status() -> ToolStatus { let tool_name = MEMTRACK_COMMAND.to_string(); @@ -70,16 +71,10 @@ pub fn get_memtrack_status() -> ToolStatus { } pub async fn install_memtrack() -> Result<()> { - let get_memtrack_installer_url = || { - format!( - "https://github.com/CodSpeedHQ/codspeed/releases/download/memtrack-v{MEMTRACK_CODSPEED_VERSION}/memtrack-installer.sh" - ) - }; - ensure_binary_installed( MEMTRACK_COMMAND, MEMTRACK_CODSPEED_VERSION, - get_memtrack_installer_url, + PinnedBinary::MemtrackInstaller, ) .await } diff --git a/src/executor/orchestrator.rs b/src/executor/orchestrator.rs index 0ab89209..10f58d50 100644 --- a/src/executor/orchestrator.rs +++ b/src/executor/orchestrator.rs @@ -1,6 +1,7 @@ use super::{ExecutionContext, ExecutorName, get_executor_from_mode, run_executor}; use crate::api_client::CodSpeedAPIClient; use crate::binary_installer::ensure_binary_installed; +use crate::binary_pins::{self, PinnedBinary}; use crate::cli::exec::multi_targets; use crate::cli::run::logger::Logger; use crate::executor::config::BenchmarkTarget; @@ -18,7 +19,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; pub const EXEC_HARNESS_COMMAND: &str = "exec-harness"; -pub const EXEC_HARNESS_VERSION: &str = "1.3.0"; +pub const EXEC_HARNESS_VERSION: &str = binary_pins::EXEC_HARNESS_VERSION; /// Shared orchestration state created once per CLI invocation. /// @@ -82,11 +83,11 @@ impl Orchestrator { .collect(); if !exec_targets.is_empty() { - ensure_binary_installed(EXEC_HARNESS_COMMAND, EXEC_HARNESS_VERSION, || { - format!( - "https://github.com/CodSpeedHQ/codspeed/releases/download/exec-harness-v{EXEC_HARNESS_VERSION}/exec-harness-installer.sh" - ) - }) + ensure_binary_installed( + EXEC_HARNESS_COMMAND, + EXEC_HARNESS_VERSION, + PinnedBinary::ExecHarnessInstaller, + ) .await?; let pipe_cmd = multi_targets::build_exec_targets_pipe_command(&exec_targets)?; diff --git a/src/executor/valgrind/setup.rs b/src/executor/valgrind/setup.rs index b04a4c6c..c49a36da 100644 --- a/src/executor/valgrind/setup.rs +++ b/src/executor/valgrind/setup.rs @@ -1,21 +1,19 @@ -use crate::cli::run::helpers::download_file; +use crate::binary_pins::PinnedBinary; +use crate::cli::run::helpers::download_pinned_file; use crate::executor::helpers::apt; use crate::executor::{ToolInstallStatus, ToolStatus}; use crate::prelude::*; use crate::system::{LinuxDistribution, SupportedOs, SystemInfo}; -use crate::{ - VALGRIND_CODSPEED_DEB_VERSION, VALGRIND_CODSPEED_VERSION, VALGRIND_CODSPEED_VERSION_STRING, -}; +use crate::{VALGRIND_CODSPEED_VERSION, VALGRIND_CODSPEED_VERSION_STRING}; use semver::Version; use std::{env, path::Path, process::Command}; -use url::Url; -pub(super) fn get_codspeed_valgrind_filename(system_info: &SystemInfo) -> Result { +fn get_codspeed_valgrind_target(system_info: &SystemInfo) -> Result<(&'static str, &'static str)> { let SupportedOs::Linux(distro) = &system_info.os else { bail!("Unsupported system"); }; - let (deb_ubuntu_version, architecture) = match (distro, system_info.arch.as_str()) { + let target = match (distro, system_info.arch.as_str()) { (LinuxDistribution::Ubuntu { version }, "x86_64") | (LinuxDistribution::Debian { version }, "x86_64") if version == "22.04" || version == "12" => @@ -37,16 +35,19 @@ pub(super) fn get_codspeed_valgrind_filename(system_info: &SystemInfo) -> Result _ => bail!("Unsupported system"), }; - Ok(format!( - "valgrind_{}_ubuntu-{}_{}.deb", - VALGRIND_CODSPEED_DEB_VERSION.as_str(), - deb_ubuntu_version, - architecture - )) + Ok(target) +} + +fn get_codspeed_valgrind_binary(system_info: &SystemInfo) -> Result { + let (distro_version, arch) = get_codspeed_valgrind_target(system_info)?; + Ok(PinnedBinary::ValgrindDeb { + distro_version, + arch, + }) } pub(super) fn is_codspeed_valgrind_installation_supported(system_info: &SystemInfo) -> bool { - get_codspeed_valgrind_filename(system_info).is_ok() + get_codspeed_valgrind_target(system_info).is_ok() } /// Parse a valgrind version string and extract the semantic version. @@ -168,13 +169,9 @@ pub async fn install_valgrind( is_valgrind_installed, || async { debug!("Installing valgrind"); - let valgrind_deb_url = format!( - "https://github.com/CodSpeedHQ/valgrind-codspeed/releases/download/{}/{}", - VALGRIND_CODSPEED_DEB_VERSION.as_str(), - get_codspeed_valgrind_filename(system_info)? - ); + let binary = get_codspeed_valgrind_binary(system_info)?; let deb_path = env::temp_dir().join("valgrind-codspeed.deb"); - download_file(&Url::parse(valgrind_deb_url.as_str()).unwrap(), &deb_path).await?; + download_pinned_file(binary, &deb_path).await?; apt::install(system_info, &[deb_path.to_str().unwrap()])?; // Return package names for caching @@ -191,7 +188,7 @@ mod tests { use super::*; #[test] - fn test_system_info_to_codspeed_valgrind_version_ubuntu() { + fn test_system_info_to_codspeed_valgrind_url_ubuntu() { let system_info = SystemInfo { os: SupportedOs::Linux(LinuxDistribution::Ubuntu { version: "22.04".into(), @@ -200,13 +197,13 @@ mod tests { ..SystemInfo::test() }; assert_snapshot!( - get_codspeed_valgrind_filename(&system_info).unwrap(), - @"valgrind_3.26.0-0codspeed0_ubuntu-22.04_amd64.deb" + get_codspeed_valgrind_binary(&system_info).unwrap().url(), + @"https://github.com/CodSpeedHQ/valgrind-codspeed/releases/download/3.26.0-0codspeed0/valgrind_3.26.0-0codspeed0_ubuntu-22.04_amd64.deb" ); } #[test] - fn test_system_info_to_codspeed_valgrind_version_ubuntu_24() { + fn test_system_info_to_codspeed_valgrind_url_ubuntu_24() { let system_info = SystemInfo { os: SupportedOs::Linux(LinuxDistribution::Ubuntu { version: "24.04".into(), @@ -215,13 +212,13 @@ mod tests { ..SystemInfo::test() }; assert_snapshot!( - get_codspeed_valgrind_filename(&system_info).unwrap(), - @"valgrind_3.26.0-0codspeed0_ubuntu-24.04_amd64.deb" + get_codspeed_valgrind_binary(&system_info).unwrap().url(), + @"https://github.com/CodSpeedHQ/valgrind-codspeed/releases/download/3.26.0-0codspeed0/valgrind_3.26.0-0codspeed0_ubuntu-24.04_amd64.deb" ); } #[test] - fn test_system_info_to_codspeed_valgrind_version_debian() { + fn test_system_info_to_codspeed_valgrind_url_debian() { let system_info = SystemInfo { os: SupportedOs::Linux(LinuxDistribution::Debian { version: "12".into(), @@ -230,13 +227,13 @@ mod tests { ..SystemInfo::test() }; assert_snapshot!( - get_codspeed_valgrind_filename(&system_info).unwrap(), - @"valgrind_3.26.0-0codspeed0_ubuntu-22.04_amd64.deb" + get_codspeed_valgrind_binary(&system_info).unwrap().url(), + @"https://github.com/CodSpeedHQ/valgrind-codspeed/releases/download/3.26.0-0codspeed0/valgrind_3.26.0-0codspeed0_ubuntu-22.04_amd64.deb" ); } #[test] - fn test_system_info_to_codspeed_valgrind_version_ubuntu_arm() { + fn test_system_info_to_codspeed_valgrind_url_ubuntu_arm() { let system_info = SystemInfo { os: SupportedOs::Linux(LinuxDistribution::Ubuntu { version: "22.04".into(), @@ -245,31 +242,31 @@ mod tests { ..SystemInfo::test() }; assert_snapshot!( - get_codspeed_valgrind_filename(&system_info).unwrap(), - @"valgrind_3.26.0-0codspeed0_ubuntu-22.04_arm64.deb" + get_codspeed_valgrind_binary(&system_info).unwrap().url(), + @"https://github.com/CodSpeedHQ/valgrind-codspeed/releases/download/3.26.0-0codspeed0/valgrind_3.26.0-0codspeed0_ubuntu-22.04_arm64.deb" ); } #[test] - fn test_codspeed_valgrind_filename_unsupported_os() { + fn test_codspeed_valgrind_unsupported_os() { let system_info = SystemInfo { os: SupportedOs::Macos { version: "14.0".into(), }, ..SystemInfo::test() }; - assert!(get_codspeed_valgrind_filename(&system_info).is_err()); + assert!(get_codspeed_valgrind_binary(&system_info).is_err()); } #[test] - fn test_codspeed_valgrind_filename_unsupported_distro() { + fn test_codspeed_valgrind_unsupported_distro() { let system_info = SystemInfo { os: SupportedOs::Linux(LinuxDistribution::Ubuntu { version: "20.04".into(), }), ..SystemInfo::test() }; - assert!(get_codspeed_valgrind_filename(&system_info).is_err()); + assert!(get_codspeed_valgrind_binary(&system_info).is_err()); } #[test] diff --git a/src/instruments/mongo_tracer.rs b/src/instruments/mongo_tracer.rs index c16007a6..7f10d21d 100644 --- a/src/instruments/mongo_tracer.rs +++ b/src/instruments/mongo_tracer.rs @@ -11,8 +11,9 @@ use reqwest::Client; use tokio::fs; use url::Url; -use crate::{MONGODB_TRACER_VERSION, cli::run::helpers::get_env_variable}; -use crate::{cli::run::helpers::download_file, prelude::*}; +use crate::binary_pins::PinnedBinary; +use crate::cli::run::helpers::{download_pinned_file, get_env_variable}; +use crate::prelude::*; use super::MongoDBConfig; @@ -227,16 +228,8 @@ impl MongoTracer { pub async fn install_mongodb_tracer() -> Result<()> { debug!("Installing mongodb-tracer"); - // TODO: release the tracer and update this url - let installer_url = format!( - "https://codspeed-public-assets.s3.eu-west-1.amazonaws.com/mongo-tracer/{MONGODB_TRACER_VERSION}/cs-mongo-tracer-installer.sh" - ); let installer_path = env::temp_dir().join("cs-mongo-tracer-installer.sh"); - download_file( - &Url::parse(installer_url.as_str()).unwrap(), - &installer_path, - ) - .await?; + download_pinned_file(PinnedBinary::MongoTracerInstaller, &installer_path).await?; let output = Command::new("bash") .arg(installer_path.to_str().unwrap()) diff --git a/src/lib.rs b/src/lib.rs index 6ddfdf13..408f437d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ mod api_client; mod binary_installer; +mod binary_pins; pub mod cli; mod config; mod executor; @@ -24,12 +25,7 @@ use semver::Version; use std::sync::LazyLock; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const MONGODB_TRACER_VERSION: &str = "cs-mongo-tracer-v0.2.0"; pub const VALGRIND_CODSPEED_VERSION: Version = Version::new(3, 26, 0); -pub const VALGRIND_CODSPEED_DEB_REVISION_SUFFIX: &str = "0codspeed0"; pub static VALGRIND_CODSPEED_VERSION_STRING: LazyLock = LazyLock::new(|| format!("{VALGRIND_CODSPEED_VERSION}.codspeed")); -pub static VALGRIND_CODSPEED_DEB_VERSION: LazyLock = LazyLock::new(|| { - format!("{VALGRIND_CODSPEED_VERSION}-{VALGRIND_CODSPEED_DEB_REVISION_SUFFIX}") -}); From 585b6dd6cf6a0d95861f047ddbd0bcb4e599185d Mon Sep 17 00:00:00 2001 From: Arthur Pastel Date: Wed, 20 May 2026 21:19:36 -0700 Subject: [PATCH 2/2] test: verify every pinned binary's sha256 against its live URL Add a network-bound test that downloads each `PinnedBinary` variant (valgrind for every supported distro/arch combination, plus the three installer scripts) and asserts the bytes match the declared SHA-256 via `download_pinned_file`. Gated behind `GITHUB_ACTIONS` like the other network/sudo tests, so it runs in CI but is opt-in locally. This catches a stale or mistyped hash before the release rather than at install time on a user's machine. Document the local invocation in CONTRIBUTING.md alongside the regeneration workflow. Co-Authored-By: Claude --- CONTRIBUTING.md | 8 +++++++ src/binary_pins.rs | 58 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7e7de2e0..0cf5627e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -53,6 +53,14 @@ curl -sL '' | sha256sum For valgrind, that is one hash per supported `(distro_version, arch)` combination. +After updating, run the network-bound verification test that downloads every pinned URL and checks the bytes against the declared hash: + +```bash +GITHUB_ACTIONS=true cargo test --lib binary_pins::tests::all_pinned_binaries_match_their_declared_sha256 +``` + +This is also run in CI, but running it locally before opening the PR avoids a release-time round trip if a hash is wrong. + ### Releasing the Main Runner The main runner (`codspeed-runner`) should be released after ensuring all dependency versions are correct. diff --git a/src/binary_pins.rs b/src/binary_pins.rs index b6ac6353..e8ca501a 100644 --- a/src/binary_pins.rs +++ b/src/binary_pins.rs @@ -42,6 +42,33 @@ impl PinnedBinary { } } + /// Every `PinnedBinary` instance the runner can produce at runtime, used + /// by the verification test to catch a stale or mistyped hash before + /// release. Update this whenever a new variant or valgrind target is + /// added. + #[cfg(test)] + pub const ALL: &'static [PinnedBinary] = &[ + PinnedBinary::ValgrindDeb { + distro_version: "22.04", + arch: "amd64", + }, + PinnedBinary::ValgrindDeb { + distro_version: "24.04", + arch: "amd64", + }, + PinnedBinary::ValgrindDeb { + distro_version: "22.04", + arch: "arm64", + }, + PinnedBinary::ValgrindDeb { + distro_version: "24.04", + arch: "arm64", + }, + PinnedBinary::MemtrackInstaller, + PinnedBinary::ExecHarnessInstaller, + PinnedBinary::MongoTracerInstaller, + ]; + pub fn sha256(&self) -> &'static str { match self { PinnedBinary::ValgrindDeb { @@ -74,3 +101,34 @@ impl PinnedBinary { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::cli::run::helpers::download_pinned_file; + use tempfile::NamedTempFile; + + // Network-bound: downloads every pinned URL and asserts its bytes hash to + // the declared SHA-256. Skipped locally; CI sets `GITHUB_ACTIONS=true`. + // Run after bumping a version to make sure the release won't ship a stale + // or mistyped hash. + #[test_with::env(GITHUB_ACTIONS)] + #[tokio::test(flavor = "multi_thread")] + async fn all_pinned_binaries_match_their_declared_sha256() { + let results = + futures::future::join_all(PinnedBinary::ALL.iter().map(|binary| async move { + let temp = NamedTempFile::new().expect("failed to create temp file"); + download_pinned_file(*binary, temp.path()) + .await + .map_err(|e| format!("{binary:?} ({}): {e}", binary.url())) + })) + .await; + + let failures: Vec<_> = results.into_iter().filter_map(Result::err).collect(); + assert!( + failures.is_empty(), + "pinned binaries failed verification:\n - {}", + failures.join("\n - "), + ); + } +}