diff --git a/crates/vt/docs/task-cache.md b/crates/vt/docs/task-cache.md index c39fa45fd..64050cd62 100644 --- a/crates/vt/docs/task-cache.md +++ b/crates/vt/docs/task-cache.md @@ -167,10 +167,11 @@ The cached execution result: ```rust pub struct CacheEntryValue { - pub post_run_fingerprint: PostRunFingerprint, + pub input_fingerprints: InputFingerprints, + pub tracked_env_fingerprints: TrackedEnvFingerprints, pub std_outputs: Arc<[StdOutput]>, pub duration: Duration, - pub globbed_inputs: BTreeMap, + pub output_archive: Option, } ``` @@ -357,10 +358,11 @@ Cache entries are serialized using `bincode` for efficient storage. │ 3. Create CacheEntryValue │ │ ──────────────────────────── │ │ CacheEntryValue { │ -│ post_run_fingerprint, │ +│ input_fingerprints, │ +│ tracked_env_fingerprints, │ │ std_outputs, │ │ duration, │ -│ globbed_inputs, │ +│ output_archive, │ │ } │ │ │ │ │ ▼ │ @@ -571,7 +573,8 @@ crates/vt/src/session/ │ └── display.rs # Cache status display formatting ├── execute/ │ ├── mod.rs # execute_spawn, SpawnOutcome -│ ├── fingerprint.rs # PostRunFingerprint, PathFingerprint, DirEntryKind +│ ├── fingerprint.rs # InputFingerprints, PathFingerprint, InputChange +│ ├── post_run.rs # TrackedEnvFingerprints (tracked env validation) │ └── spawn.rs # spawn_with_tracking, fspy integration └── reporter/ └── mod.rs # Reporter traits for cache hit/miss display diff --git a/crates/vt/src/session/cache/display.rs b/crates/vt/src/session/cache/display.rs index 87d86958a..5ae8e4bf9 100644 --- a/crates/vt/src/session/cache/display.rs +++ b/crates/vt/src/session/cache/display.rs @@ -250,7 +250,7 @@ mod tests { fn inline_tracked_env_mismatch_preserves_kind() { let added = CacheStatus::Miss(CacheMiss::FingerprintMismatch( FingerprintMismatch::TrackedEnvQueryChanged { - query: crate::session::execute::fingerprint::TrackedEnvQuery::Glob(Str::from( + query: crate::session::execute::post_run::TrackedEnvQuery::Glob(Str::from( "PROBE_*", )), mismatch: EnvMismatch::Added { name: Str::from("PROBE_C") }, diff --git a/crates/vt/src/session/cache/mod.rs b/crates/vt/src/session/cache/mod.rs index 06780e188..91954ec4e 100644 --- a/crates/vt/src/session/cache/mod.rs +++ b/crates/vt/src/session/cache/mod.rs @@ -25,9 +25,11 @@ use wincode::{ io::{Reader, Writer}, }; +pub use super::execute::fingerprint::InputChangeKind; use super::execute::{ - fingerprint::{PostRunFingerprint, TrackedEnvQuery}, + fingerprint::{InputChange, InputFingerprints}, pipe::StdOutput, + post_run::{PostRunMismatch, TrackedEnvFingerprints, TrackedEnvQuery}, }; const TASK_CACHE_PREALLOCATION_SIZE_LIMIT: usize = 256 * 1024 * 1024; @@ -118,19 +120,19 @@ unsafe impl<'de, C: ConfigCore> SchemaRead<'de, C> for DurationSchema { /// Cached execution result for a task. /// -/// Contains the post-run fingerprint (from fspy), captured outputs, -/// execution duration, and explicit input file hashes. +/// Contains the run's input fingerprints and tracked env state, captured +/// outputs, and execution duration. #[derive(Debug, SchemaWrite, SchemaRead, Serialize)] pub struct CacheEntryValue { - pub post_run_fingerprint: PostRunFingerprint, + /// Fingerprints of everything the cached run read. Checked against the + /// filesystem at lookup to decide whether the entry is still valid. + pub input_fingerprints: InputFingerprints, + /// Env vars and bulk env queries observed by runner-aware tools during + /// the run. Checked against the current env context at lookup. + pub tracked_env_fingerprints: TrackedEnvFingerprints, pub std_outputs: Arc<[StdOutput]>, #[wincode(with = "DurationSchema")] pub duration: Duration, - /// Hashes of explicit input files computed from positive globs. - /// Files matching negative globs are already filtered out. - /// Path is relative to workspace root, value is `xxHash3_64` of file content. - /// Stored in the value (not the key) so changes can be detected and reported. - pub globbed_inputs: BTreeMap, /// Filename of the output archive (e.g. `{uuid}.tar.zst`) stored alongside /// `cache.db` in the cache directory. `None` if no output files were produced. pub output_archive: Option, @@ -151,16 +153,6 @@ pub enum CacheMiss { FingerprintMismatch(FingerprintMismatch), } -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub enum InputChangeKind { - /// File content changed but path is the same - ContentModified, - /// New file or folder added - Added, - /// Existing file or folder removed - Removed, -} - /// A single env var difference between a stored fingerprint and the current /// environment. /// @@ -240,11 +232,15 @@ pub enum FingerprintMismatch { }, } -impl From for FingerprintMismatch { - fn from(mismatch: crate::session::execute::fingerprint::PostRunMismatch) -> Self { - use crate::session::execute::fingerprint::PostRunMismatch; +impl From for FingerprintMismatch { + fn from(change: InputChange) -> Self { + Self::InputChanged { kind: change.kind, path: change.path } + } +} + +impl From for FingerprintMismatch { + fn from(mismatch: PostRunMismatch) -> Self { match mismatch { - PostRunMismatch::Input { kind, path } => Self::InputChanged { kind, path }, PostRunMismatch::TrackedEnv(mismatch) => Self::TrackedEnvChanged(mismatch), PostRunMismatch::TrackedEnvQuery { query, mismatch } => { Self::TrackedEnvQueryChanged { query, mismatch } @@ -274,7 +270,7 @@ pub fn split_path(path: &str) -> (Option<&str>, &str) { /// its own cache warm across branch switches, and a cache from a different /// version is simply ignored (it lives in a directory this build never looks /// at) rather than aborting the run. Bumping the version starts a fresh cache. -const CACHE_SCHEMA_VERSION: u32 = 18; +const CACHE_SCHEMA_VERSION: u32 = 19; /// Name of the per-version subdirectory (e.g. `v14`) under the task-cache /// directory that holds the database and output archives for the current @@ -334,17 +330,19 @@ impl ExecutionCache { // Try to find the cache entry by key (spawn fingerprint + input config) if let Some(cache_value) = self.get_by_cache_key(&cache_key).await? { - // Validate explicit globbed inputs against the stored values - if let Some(mismatch) = - detect_globbed_input_change(&cache_value.globbed_inputs, globbed_inputs) + // Validate the stored input fingerprints against the filesystem: + // the listed-inputs snapshot against the fresh one, then each + // discovered input against the disk. + if let Some(change) = + cache_value.input_fingerprints.find_change(globbed_inputs, workspace_root)? { - return Ok(Err(CacheMiss::FingerprintMismatch(mismatch))); + return Ok(Err(CacheMiss::FingerprintMismatch(change.into()))); } - // Validate post-run fingerprint (inferred inputs + tracked envs) + // Validate the tracked env state against the current env context. if let Some(mismatch) = cache_value - .post_run_fingerprint - .validate(workspace_root, &cache_metadata.unfiltered_envs)? + .tracked_env_fingerprints + .validate_envs(&cache_metadata.unfiltered_envs)? { return Ok(Err(CacheMiss::FingerprintMismatch(mismatch.into()))); } @@ -419,60 +417,6 @@ impl ExecutionCache { } } -/// Compare stored and current globbed inputs, returning the first changed path. -/// Both maps are `BTreeMap` so we iterate them in sorted lockstep. -fn detect_globbed_input_change( - stored: &BTreeMap, - current: &BTreeMap, -) -> Option { - let mut stored_iter = stored.iter(); - let mut current_iter = current.iter(); - let mut s = stored_iter.next(); - let mut c = current_iter.next(); - - loop { - match (s, c) { - (None, None) => return None, - (Some((sp, _)), None) => { - return Some(FingerprintMismatch::InputChanged { - kind: InputChangeKind::Removed, - path: sp.clone(), - }); - } - (None, Some((cp, _))) => { - return Some(FingerprintMismatch::InputChanged { - kind: InputChangeKind::Added, - path: cp.clone(), - }); - } - (Some((sp, sh)), Some((cp, ch))) => match sp.cmp(cp) { - std::cmp::Ordering::Equal => { - if sh != ch { - return Some(FingerprintMismatch::InputChanged { - kind: InputChangeKind::ContentModified, - path: sp.clone(), - }); - } - s = stored_iter.next(); - c = current_iter.next(); - } - std::cmp::Ordering::Less => { - return Some(FingerprintMismatch::InputChanged { - kind: InputChangeKind::Removed, - path: sp.clone(), - }); - } - std::cmp::Ordering::Greater => { - return Some(FingerprintMismatch::InputChanged { - kind: InputChangeKind::Added, - path: cp.clone(), - }); - } - }, - } - } -} - // Basic database operations impl ExecutionCache { #[expect( diff --git a/crates/vt/src/session/execute/cache_update.rs b/crates/vt/src/session/execute/cache_update.rs index f13d3f1df..88089cfc4 100644 --- a/crates/vt/src/session/execute/cache_update.rs +++ b/crates/vt/src/session/execute/cache_update.rs @@ -11,8 +11,9 @@ use vt_str::Str; use super::{ CacheState, - fingerprint::{PathRead, PostRunFingerprint, TrackedEnvQuery}, + fingerprint::{InputFingerprints, PathRead, fingerprint_discovered}, glob, + post_run::{TrackedEnvFingerprints, TrackedEnvQuery}, spawn::ChildOutcome, }; use crate::{ @@ -132,19 +133,13 @@ pub(super) async fn update_cache( } }; - // Paths already in globbed_inputs are skipped: the overlap check above - // guarantees no input modification, so the prerun hash is the correct - // post-exec hash. + // Fingerprint the discovered inputs (traced reads not already in the + // globbed snapshot: the overlap check above guarantees no input + // modification, so the prerun hash is the correct post-exec hash). let empty_path_reads = HashMap::default(); let path_reads = fspy_outcome.as_ref().map_or(&empty_path_reads, |o| &o.path_reads); - let post_run_fingerprint = match PostRunFingerprint::create( - path_reads, - workspace_root, - &globbed_inputs, - tracked_envs, - tracked_env_queries, - ) { - Ok(fingerprint) => fingerprint, + let discovered = match fingerprint_discovered(path_reads, workspace_root, &globbed_inputs) { + Ok(discovered) => discovered, Err(err) => { return ( CacheUpdateStatus::NotUpdated(CacheNotUpdatedReason::CacheDisabled), @@ -169,10 +164,10 @@ pub(super) async fn update_cache( }; let new_cache_value = CacheEntryValue { - post_run_fingerprint, + input_fingerprints: InputFingerprints::new(globbed_inputs, discovered), + tracked_env_fingerprints: TrackedEnvFingerprints { tracked_envs, tracked_env_queries }, std_outputs: std_outputs.into(), duration, - globbed_inputs, output_archive, }; match cache.update(metadata, new_cache_value, cache_dir).await { diff --git a/crates/vt/src/session/execute/fingerprint.rs b/crates/vt/src/session/execute/fingerprint.rs index a8011ce2c..721f5ab81 100644 --- a/crates/vt/src/session/execute/fingerprint.rs +++ b/crates/vt/src/session/execute/fingerprint.rs @@ -1,36 +1,19 @@ -//! Post-run fingerprinting for execution caching. -//! -//! This module provides types and functions for creating and validating -//! fingerprints of file system state after task execution. +//! Per-path fingerprints and the run's input-fingerprint record. use std::{ collections::BTreeMap, - ffi::OsStr, fs::File, io::{self, BufRead}, sync::Arc, }; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; use vt_path::{AbsolutePath, RelativePathBuf}; -use vt_plan::cache_metadata::EnvValueHash; use vt_str::Str; use wincode::{SchemaRead, SchemaWrite}; -use crate::{ - collections::HashMap, - session::cache::{EnvMismatch, InputChangeKind}, -}; - -#[derive( - SchemaWrite, SchemaRead, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, -)] -pub enum TrackedEnvQuery { - Glob(Str), - Prefix(Str), -} +use crate::collections::HashMap; /// Path read access info #[derive(Debug, Clone, Copy)] @@ -38,46 +21,6 @@ pub struct PathRead { pub read_dir_entries: bool, } -/// Post-run fingerprint capturing file state after execution. -/// Used to validate whether cached outputs are still valid. -#[derive(SchemaWrite, SchemaRead, Debug, Default, Serialize)] -pub struct PostRunFingerprint { - /// Paths inferred from fspy during execution with their content fingerprints. - /// Only populated when `input_config.includes_auto` is true. - pub inferred_inputs: HashMap, - - /// Env vars observed via runner-aware IPC `getEnv` with `tracked: true`. - /// Key is the env name; value is the env value hash at execution time, or - /// `None` if unset. Validated at cache lookup against the same plan env - /// context that served the original request. - pub tracked_envs: BTreeMap>, - - /// Bulk env queries (`getEnvs`) made with `tracked: true`. - /// Outer key is the query, inner map is the match-set at execution time - /// (name -> value hash). Validated at cache lookup by re-matching against - /// the current env context and comparing the resulting set. - /// - /// Non-UTF-8 env names are never matched, saved, or treated as errors: - /// they are not returned to the client, so their existence cannot affect - /// task behavior. Values are stricter. A matched env must have a UTF-8 - /// value; the JS client errors when querying a matched non-UTF-8 value, - /// and cache-hit validation treats a currently matched non-UTF-8 value as - /// a changed mismatch so stale cached output is not replayed. - pub tracked_env_queries: BTreeMap>, -} - -/// A mismatch between the stored post-run fingerprint and the current state. -#[derive(Debug, Clone)] -pub enum PostRunMismatch { - /// An inferred input file or directory changed. - Input { kind: InputChangeKind, path: RelativePathBuf }, - /// A tool-tracked env var changed value, appeared, or disappeared. - TrackedEnv(EnvMismatch), - /// A tool-tracked bulk env query's match-set changed between runs. Carries - /// the first differing entry in env-name order. - TrackedEnvQuery { query: TrackedEnvQuery, mismatch: EnvMismatch }, -} - /// Fingerprint for a single path (file or directory) #[derive(SchemaWrite, SchemaRead, PartialEq, Eq, Debug, Clone, Serialize, Deserialize)] pub enum PathFingerprint { @@ -100,200 +43,127 @@ pub enum DirEntryKind { Symlink, } -impl PostRunFingerprint { - /// Creates a new fingerprint from path accesses after task execution. - /// - /// Negative glob filtering is done upstream (see - /// [`super::tracked_accesses::TrackedPathAccesses::from_raw`]). - /// Paths already present in `globbed_inputs` are skipped — they are - /// already tracked by the prerun glob fingerprint, and the read-write - /// overlap check in `execute_spawn` guarantees the task did not modify - /// them, so the prerun hash is still correct. - /// - /// # Arguments - /// * `inferred_path_reads` - Map of paths that were read during execution (from fspy) - /// * `base_dir` - Workspace root for resolving relative paths - /// * `globbed_inputs` - Prerun glob fingerprint; paths here are skipped - /// * `tracked_envs` - Tool-requested env vars (name -> value hash), validated on lookup - /// * `tracked_env_queries` - Tool-requested bulk env queries (query -> match-set hashes) - #[tracing::instrument(level = "debug", skip_all, name = "create_post_run_fingerprint")] - pub fn create( - inferred_path_reads: &HashMap, - base_dir: &AbsolutePath, - globbed_inputs: &BTreeMap, - tracked_envs: BTreeMap>, - tracked_env_queries: BTreeMap>, - ) -> anyhow::Result { - let inferred_inputs = inferred_path_reads - .par_iter() - .filter(|(path, _)| !globbed_inputs.contains_key(*path)) - .map(|(relative_path, path_read)| { - let full_path = Arc::::from(base_dir.join(relative_path)); - let fingerprint = fingerprint_path(&full_path, *path_read)?; - Ok((relative_path.clone(), fingerprint)) - }) - .collect::>>()?; - - Ok(Self { inferred_inputs, tracked_envs, tracked_env_queries }) - } - - /// Validates the fingerprint against current filesystem state and the - /// unfiltered env context used by runner-aware IPC. `unfiltered_envs` must - /// be the same plan env context that served the original `getEnv` request, - /// not the filtered env passed to the spawned process. - /// - /// Returns `Some(mismatch)` if anything changed, `None` if all valid. - /// Returns an error if a tracked env is currently present but cannot be - /// represented as UTF-8; treating that value as unset would make cache - /// validation unsound. - #[tracing::instrument(level = "debug", skip_all, name = "validate_post_run_fingerprint")] - pub fn validate( - &self, - base_dir: &AbsolutePath, - unfiltered_envs: &FxHashMap, Arc>, - ) -> anyhow::Result> { - let input_mismatch = self.inferred_inputs.par_iter().find_map_any( - |(input_relative_path, path_fingerprint)| { - let input_full_path = Arc::::from(base_dir.join(input_relative_path)); - let path_read = PathRead { - read_dir_entries: matches!(path_fingerprint, PathFingerprint::Folder(Some(_))), - }; - let current_path_fingerprint = match fingerprint_path(&input_full_path, path_read) { - Ok(ok) => ok, - Err(err) => return Some(Err(err)), - }; - if path_fingerprint == ¤t_path_fingerprint { - None - } else { - let (kind, entry_name) = - determine_change_kind(path_fingerprint, ¤t_path_fingerprint); - let path = if let Some(name) = entry_name { - // For folder changes, build `dir/entry` path - let entry = match RelativePathBuf::new(name.as_str()) { - Ok(p) => p, - Err(e) => return Some(Err(e.into())), - }; - input_relative_path.as_relative_path().join(entry) - } else { - input_relative_path.clone() - }; - Some(Ok(PostRunMismatch::Input { kind, path })) - } - }, - ); - if let Some(result) = input_mismatch { - return result.map(Some); - } - - for (name, stored_value) in &self.tracked_envs { - let current_value = unfiltered_envs - .get(OsStr::new(name.as_str())) - .map(|value| { - let value_str = value.to_str().ok_or_else(|| { - anyhow::anyhow!("tracked env value for {name} is not valid UTF-8") - })?; - Ok::<_, anyhow::Error>(EnvValueHash::new(value_str)) - }) - .transpose()?; - if let Some(mismatch) = - EnvMismatch::compare(name, stored_value.as_ref(), current_value.as_ref()) - { - return Ok(Some(PostRunMismatch::TrackedEnv(mismatch))); - } - } - - for (query, stored_matches) in &self.tracked_env_queries { - let current_matches = match match_env_query(query, unfiltered_envs)? { - EnvQueryValidation::Matches(matches) => matches, - EnvQueryValidation::NonUtf8Value(mismatch) => { - return Ok(Some(PostRunMismatch::TrackedEnvQuery { - query: query.clone(), - mismatch, - })); - } - }; - if let Some(mismatch) = first_env_glob_mismatch(stored_matches, ¤t_matches) { - return Ok(Some(PostRunMismatch::TrackedEnvQuery { - query: query.clone(), - mismatch, - })); - } - } +/// How an input changed since a previous run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum InputChangeKind { + /// File content changed but path is the same + ContentModified, + /// New file or folder added + Added, + /// Existing file or folder removed + Removed, +} - Ok(None) - } +/// The first input found to differ from a previous run. +#[derive(Debug, Clone)] +pub struct InputChange { + pub kind: InputChangeKind, + pub path: RelativePathBuf, } -/// Build the current match-set for `query` by enumerating the given env -/// snapshot and keeping matching UTF-8 names. If a matching env has a non-UTF-8 -/// value, return a changed mismatch so the stale cache entry is not replayed. -fn match_env_query( - query: &TrackedEnvQuery, - envs: &FxHashMap, Arc>, -) -> anyhow::Result { - Ok(match query { - TrackedEnvQuery::Glob(pattern) => { - let glob = vt_glob::env::EnvGlob::new(pattern.as_str())?; - collect_matching_envs(envs, |name| glob.is_match(name)) - } - TrackedEnvQuery::Prefix(prefix) => { - collect_matching_envs(envs, |name| env_name_starts_with(name, prefix.as_str())) - } - }) +/// Fingerprints of everything a task run read. +/// +/// Opaque: produced at the end of a run, stored by the caller, and handed +/// back at the start of a later run to detect input changes. +#[derive(SchemaWrite, SchemaRead, PartialEq, Eq, Debug, Default, Serialize)] +pub struct InputFingerprints { + /// Content hashes of the explicitly-listed inputs, captured before the + /// run (the pre-run snapshot). The read-write-overlap check guarantees + /// the task did not modify them, so they double as the post-run state. + snapshot: BTreeMap, + /// Fingerprints of the inputs the run discovered while it ran (traced + /// reads not already covered by the snapshot). + discovered: HashMap, } -fn collect_matching_envs( - envs: &FxHashMap, Arc>, - is_match: impl Fn(&str) -> bool, -) -> EnvQueryValidation { - let mut matches = BTreeMap::new(); - for (name, value) in envs { - let Some(name_str) = name.to_str() else { - continue; - }; - if !is_match(name_str) { - continue; - } - let Some(value_str) = value.to_str() else { - return EnvQueryValidation::NonUtf8Value(EnvMismatch::Changed { - name: Str::from(name_str), - }); - }; - matches.insert(Str::from(name_str), EnvValueHash::new(value_str)); +impl InputFingerprints { + pub(crate) const fn new( + snapshot: BTreeMap, + discovered: HashMap, + ) -> Self { + Self { snapshot, discovered } } - EnvQueryValidation::Matches(matches) -} -enum EnvQueryValidation { - Matches(BTreeMap), - NonUtf8Value(EnvMismatch), + /// Find the first input that differs between this (stored) record and the + /// present: the stored snapshot is diffed against `current_snapshot` + /// (pure), then each discovered input is re-fingerprinted against the + /// filesystem. Check order determines which change gets reported when + /// several exist. + pub(crate) fn find_change( + &self, + current_snapshot: &BTreeMap, + base_dir: &AbsolutePath, + ) -> anyhow::Result> { + if let Some(change) = detect_snapshot_change(&self.snapshot, current_snapshot) { + return Ok(Some(change)); + } + validate_discovered(&self.discovered, base_dir) + } } -#[cfg(not(windows))] -fn env_name_starts_with(name: &str, prefix: &str) -> bool { - name.starts_with(prefix) +/// Fingerprint the inputs a run discovered: every traced read not already in +/// the snapshot. Paths in the snapshot are skipped — they are already tracked +/// by the pre-run hashes, and the read-write overlap check guarantees the task +/// did not modify them, so the pre-run hash is still correct. +pub fn fingerprint_discovered( + path_reads: &HashMap, + base_dir: &AbsolutePath, + snapshot: &BTreeMap, +) -> anyhow::Result> { + path_reads + .par_iter() + .filter(|(path, _)| !snapshot.contains_key(*path)) + .map(|(relative_path, path_read)| { + let full_path = Arc::::from(base_dir.join(relative_path)); + let fingerprint = fingerprint_path(&full_path, *path_read)?; + Ok((relative_path.clone(), fingerprint)) + }) + .collect::>>() } -#[cfg(windows)] -fn env_name_starts_with(name: &str, prefix: &str) -> bool { - let mut name_chars = name.chars(); - for prefix_char in prefix.chars() { - let Some(name_char) = name_chars.next() else { - return false; +/// Re-fingerprint each stored discovered input against the current filesystem +/// state, returning the first mismatch found (parallel, so the pick among +/// several concurrent mismatches is nondeterministic — intentional). +fn validate_discovered( + discovered: &HashMap, + base_dir: &AbsolutePath, +) -> anyhow::Result> { + let change = discovered.par_iter().find_map_any(|(input_relative_path, path_fingerprint)| { + let input_full_path = Arc::::from(base_dir.join(input_relative_path)); + let path_read = PathRead { + read_dir_entries: matches!(path_fingerprint, PathFingerprint::Folder(Some(_))), }; - if !name_char.eq_ignore_ascii_case(&prefix_char) { - return false; + let current_path_fingerprint = match fingerprint_path(&input_full_path, path_read) { + Ok(ok) => ok, + Err(err) => return Some(Err(err)), + }; + if path_fingerprint == ¤t_path_fingerprint { + None + } else { + let (kind, entry_name) = + determine_change_kind(path_fingerprint, ¤t_path_fingerprint); + let path = if let Some(name) = entry_name { + // For folder changes, build `dir/entry` path + let entry = match RelativePathBuf::new(name.as_str()) { + Ok(p) => p, + Err(e) => return Some(Err(e.into())), + }; + input_relative_path.as_relative_path().join(entry) + } else { + input_relative_path.clone() + }; + Some(Ok(InputChange { kind, path })) } - } - true + }); + change.transpose() } -/// Find the first deterministic difference between stored and current env -/// glob match-sets. -fn first_env_glob_mismatch( - stored: &BTreeMap, - current: &BTreeMap, -) -> Option { +/// Compare stored and current snapshot hashes, returning the first changed path. +/// Both maps are `BTreeMap` so we iterate them in sorted lockstep. +fn detect_snapshot_change( + stored: &BTreeMap, + current: &BTreeMap, +) -> Option { let mut stored_iter = stored.iter(); let mut current_iter = current.iter(); let mut s = stored_iter.next(); @@ -302,19 +172,28 @@ fn first_env_glob_mismatch( loop { match (s, c) { (None, None) => return None, - (Some((name, _)), None) => return Some(EnvMismatch::Removed { name: name.clone() }), - (None, Some((name, _))) => return Some(EnvMismatch::Added { name: name.clone() }), - (Some((sn, sv)), Some((cn, cv))) => match sn.cmp(cn) { + (Some((sp, _)), None) => { + return Some(InputChange { kind: InputChangeKind::Removed, path: sp.clone() }); + } + (None, Some((cp, _))) => { + return Some(InputChange { kind: InputChangeKind::Added, path: cp.clone() }); + } + (Some((sp, sh)), Some((cp, ch))) => match sp.cmp(cp) { std::cmp::Ordering::Equal => { - if sv != cv { - return Some(EnvMismatch::Changed { name: sn.clone() }); + if sh != ch { + return Some(InputChange { + kind: InputChangeKind::ContentModified, + path: sp.clone(), + }); } s = stored_iter.next(); c = current_iter.next(); } - std::cmp::Ordering::Less => return Some(EnvMismatch::Removed { name: sn.clone() }), + std::cmp::Ordering::Less => { + return Some(InputChange { kind: InputChangeKind::Removed, path: sp.clone() }); + } std::cmp::Ordering::Greater => { - return Some(EnvMismatch::Added { name: cn.clone() }); + return Some(InputChange { kind: InputChangeKind::Added, path: cp.clone() }); } }, } @@ -516,116 +395,41 @@ fn process_directory_unix(file: &File, path_read: PathRead) -> anyhow::Result OsString { - use std::os::unix::ffi::OsStringExt; - - OsString::from_vec(vec![0xFF]) - } - - #[cfg(windows)] - fn non_utf8_os_string() -> OsString { - use std::os::windows::ffi::OsStringExt; - - OsString::from_wide(&[0xD800]) - } - - #[test] - fn validate_errors_on_current_non_utf8_tracked_env_value() { - let mut tracked_envs = BTreeMap::new(); - tracked_envs.insert(Str::from("PROBE_ENV"), None); - let fingerprint = PostRunFingerprint { tracked_envs, ..PostRunFingerprint::default() }; - - let mut unfiltered_envs = FxHashMap::default(); - unfiltered_envs.insert( - Arc::::from(OsStr::new("PROBE_ENV")), - Arc::::from(non_utf8_os_string()), - ); - - let workspace_root = vt_path::current_dir().expect("cwd"); - let err = fingerprint - .validate(&workspace_root, &unfiltered_envs) - .expect_err("non-UTF-8 tracked env values must error"); - - assert!(err.to_string().contains("tracked env value for PROBE_ENV is not valid UTF-8")); - } - - #[test] - fn validate_reports_current_non_utf8_tracked_env_glob_value_as_changed() { - let mut tracked_env_queries = BTreeMap::new(); - tracked_env_queries.insert(TrackedEnvQuery::Glob(Str::from("PROBE_*")), BTreeMap::new()); - let fingerprint = - PostRunFingerprint { tracked_env_queries, ..PostRunFingerprint::default() }; - - let mut unfiltered_envs = FxHashMap::default(); - unfiltered_envs.insert( - Arc::::from(OsStr::new("PROBE_BAD")), - Arc::::from(non_utf8_os_string()), - ); - - let workspace_root = vt_path::current_dir().expect("cwd"); - let mismatch = - fingerprint.validate(&workspace_root, &unfiltered_envs).expect("validation succeeds"); - - match mismatch { - Some(PostRunMismatch::TrackedEnvQuery { - query, - mismatch: EnvMismatch::Changed { name }, - }) => { - assert_eq!(query, TrackedEnvQuery::Glob(Str::from("PROBE_*"))); - assert_eq!(name.as_str(), "PROBE_BAD"); - } - other => panic!("expected changed tracked env query mismatch, got {other:?}"), - } - } - + /// The opaque record must survive the cache's serialization round trip + /// with every fingerprint shape it can carry. #[test] - fn validate_tracked_env_prefix_treats_star_literally() { - let mut tracked_env_queries = BTreeMap::new(); - let mut stored_matches = BTreeMap::new(); - stored_matches.insert(Str::from("PROBE_*A"), EnvValueHash::new("literal")); - tracked_env_queries.insert(TrackedEnvQuery::Prefix(Str::from("PROBE_*")), stored_matches); - let fingerprint = - PostRunFingerprint { tracked_env_queries, ..PostRunFingerprint::default() }; - - let mut unfiltered_envs = FxHashMap::default(); - unfiltered_envs.insert( - Arc::::from(OsStr::new("PROBE_*A")), - Arc::::from(OsStr::new("literal")), - ); - unfiltered_envs.insert( - Arc::::from(OsStr::new("PROBE_XA")), - Arc::::from(OsStr::new("wildcard if interpreted as glob")), + fn input_fingerprints_wincode_round_trip() { + let mut snapshot = BTreeMap::new(); + snapshot.insert(RelativePathBuf::new("src/a.txt").unwrap(), 42u64); + + let mut entries = BTreeMap::new(); + entries.insert(Str::from("child.txt"), DirEntryKind::File); + entries.insert(Str::from("nested"), DirEntryKind::Dir); + entries.insert(Str::from("link"), DirEntryKind::Symlink); + + let mut discovered = HashMap::default(); + discovered + .insert(RelativePathBuf::new("read.txt").unwrap(), PathFingerprint::FileContentHash(7)); + discovered.insert(RelativePathBuf::new("probed").unwrap(), PathFingerprint::NotFound); + discovered + .insert(RelativePathBuf::new("opened-dir").unwrap(), PathFingerprint::Folder(None)); + discovered.insert( + RelativePathBuf::new("listed-dir").unwrap(), + PathFingerprint::Folder(Some(entries)), ); - let workspace_root = vt_path::current_dir().expect("cwd"); - let mismatch = - fingerprint.validate(&workspace_root, &unfiltered_envs).expect("validation succeeds"); - - assert!(mismatch.is_none()); - } - - #[test] - fn validate_ignores_non_utf8_tracked_env_glob_names() { - let mut tracked_env_queries = BTreeMap::new(); - tracked_env_queries.insert(TrackedEnvQuery::Glob(Str::from("PROBE_*")), BTreeMap::new()); - let fingerprint = - PostRunFingerprint { tracked_env_queries, ..PostRunFingerprint::default() }; - - let mut unfiltered_envs = FxHashMap::default(); - unfiltered_envs.insert( - Arc::::from(non_utf8_os_string()), - Arc::::from(OsStr::new("value")), - ); + let fingerprints = InputFingerprints::new(snapshot, discovered); - let workspace_root = vt_path::current_dir().expect("cwd"); - let mismatch = - fingerprint.validate(&workspace_root, &unfiltered_envs).expect("validation succeeds"); + let config = wincode::config::Configuration::default(); + let bytes = wincode::config::serialize(&fingerprints, config).unwrap(); + let back: InputFingerprints = wincode::config::deserialize_exact(&bytes, config).unwrap(); + assert_eq!(fingerprints, back); - assert!(mismatch.is_none()); + let empty = InputFingerprints::default(); + let bytes = wincode::config::serialize(&empty, config).unwrap(); + let back: InputFingerprints = wincode::config::deserialize_exact(&bytes, config).unwrap(); + assert_eq!(empty, back); } } diff --git a/crates/vt/src/session/execute/mod.rs b/crates/vt/src/session/execute/mod.rs index ae31c485e..0a9dc8986 100644 --- a/crates/vt/src/session/execute/mod.rs +++ b/crates/vt/src/session/execute/mod.rs @@ -3,6 +3,7 @@ pub mod fingerprint; pub mod glob; mod hash; pub mod pipe; +pub mod post_run; mod scheduler; pub mod spawn; #[cfg(fspy)] diff --git a/crates/vt/src/session/execute/post_run.rs b/crates/vt/src/session/execute/post_run.rs new file mode 100644 index 000000000..f504f0391 --- /dev/null +++ b/crates/vt/src/session/execute/post_run.rs @@ -0,0 +1,318 @@ +//! Post-run environment fingerprinting: env values and bulk env queries +//! observed by runner-aware tools during execution, validated again at cache +//! lookup. The filesystem half of post-run fingerprinting lives in +//! [`super::fingerprint`]. + +use std::{collections::BTreeMap, ffi::OsStr, sync::Arc}; + +use rustc_hash::FxHashMap; +use serde::{Deserialize, Serialize}; +use vt_plan::cache_metadata::EnvValueHash; +use vt_str::Str; +use wincode::{SchemaRead, SchemaWrite}; + +use crate::session::cache::EnvMismatch; + +#[derive( + SchemaWrite, SchemaRead, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, +)] +pub enum TrackedEnvQuery { + Glob(Str), + Prefix(Str), +} + +/// Env state observed by runner-aware tools during execution. +/// Used to validate whether cached outputs are still valid. +#[derive(SchemaWrite, SchemaRead, Debug, Default, Serialize)] +pub struct TrackedEnvFingerprints { + /// Env vars observed via runner-aware IPC `getEnv` with `tracked: true`. + /// Key is the env name; value is the env value hash at execution time, or + /// `None` if unset. Validated at cache lookup against the same plan env + /// context that served the original request. + pub tracked_envs: BTreeMap>, + + /// Bulk env queries (`getEnvs`) made with `tracked: true`. + /// Outer key is the query, inner map is the match-set at execution time + /// (name -> value hash). Validated at cache lookup by re-matching against + /// the current env context and comparing the resulting set. + /// + /// Non-UTF-8 env names are never matched, saved, or treated as errors: + /// they are not returned to the client, so their existence cannot affect + /// task behavior. Values are stricter. A matched env must have a UTF-8 + /// value; the JS client errors when querying a matched non-UTF-8 value, + /// and cache-hit validation treats a currently matched non-UTF-8 value as + /// a changed mismatch so stale cached output is not replayed. + pub tracked_env_queries: BTreeMap>, +} + +/// A mismatch between the stored tracked-env fingerprints and the current +/// environment. +#[derive(Debug, Clone)] +pub enum PostRunMismatch { + /// A tool-tracked env var changed value, appeared, or disappeared. + TrackedEnv(EnvMismatch), + /// A tool-tracked bulk env query's match-set changed between runs. Carries + /// the first differing entry in env-name order. + TrackedEnvQuery { query: TrackedEnvQuery, mismatch: EnvMismatch }, +} + +impl TrackedEnvFingerprints { + /// Validates the tracked env state against the unfiltered env context used + /// by runner-aware IPC. `unfiltered_envs` must be the same plan env + /// context that served the original `getEnv` request, not the filtered env + /// passed to the spawned process. + /// + /// Returns `Some(mismatch)` if anything changed, `None` if all valid. + /// Returns an error if a tracked env is currently present but cannot be + /// represented as UTF-8; treating that value as unset would make cache + /// validation unsound. + #[tracing::instrument(level = "debug", skip_all, name = "validate_tracked_envs")] + pub fn validate_envs( + &self, + unfiltered_envs: &FxHashMap, Arc>, + ) -> anyhow::Result> { + for (name, stored_value) in &self.tracked_envs { + let current_value = unfiltered_envs + .get(OsStr::new(name.as_str())) + .map(|value| { + let value_str = value.to_str().ok_or_else(|| { + anyhow::anyhow!("tracked env value for {name} is not valid UTF-8") + })?; + Ok::<_, anyhow::Error>(EnvValueHash::new(value_str)) + }) + .transpose()?; + if let Some(mismatch) = + EnvMismatch::compare(name, stored_value.as_ref(), current_value.as_ref()) + { + return Ok(Some(PostRunMismatch::TrackedEnv(mismatch))); + } + } + + for (query, stored_matches) in &self.tracked_env_queries { + let current_matches = match match_env_query(query, unfiltered_envs)? { + EnvQueryValidation::Matches(matches) => matches, + EnvQueryValidation::NonUtf8Value(mismatch) => { + return Ok(Some(PostRunMismatch::TrackedEnvQuery { + query: query.clone(), + mismatch, + })); + } + }; + if let Some(mismatch) = first_env_glob_mismatch(stored_matches, ¤t_matches) { + return Ok(Some(PostRunMismatch::TrackedEnvQuery { + query: query.clone(), + mismatch, + })); + } + } + + Ok(None) + } +} + +/// Build the current match-set for `query` by enumerating the given env +/// snapshot and keeping matching UTF-8 names. If a matching env has a non-UTF-8 +/// value, return a changed mismatch so the stale cache entry is not replayed. +fn match_env_query( + query: &TrackedEnvQuery, + envs: &FxHashMap, Arc>, +) -> anyhow::Result { + Ok(match query { + TrackedEnvQuery::Glob(pattern) => { + let glob = vt_glob::env::EnvGlob::new(pattern.as_str())?; + collect_matching_envs(envs, |name| glob.is_match(name)) + } + TrackedEnvQuery::Prefix(prefix) => { + collect_matching_envs(envs, |name| env_name_starts_with(name, prefix.as_str())) + } + }) +} + +fn collect_matching_envs( + envs: &FxHashMap, Arc>, + is_match: impl Fn(&str) -> bool, +) -> EnvQueryValidation { + let mut matches = BTreeMap::new(); + for (name, value) in envs { + let Some(name_str) = name.to_str() else { + continue; + }; + if !is_match(name_str) { + continue; + } + let Some(value_str) = value.to_str() else { + return EnvQueryValidation::NonUtf8Value(EnvMismatch::Changed { + name: Str::from(name_str), + }); + }; + matches.insert(Str::from(name_str), EnvValueHash::new(value_str)); + } + EnvQueryValidation::Matches(matches) +} + +enum EnvQueryValidation { + Matches(BTreeMap), + NonUtf8Value(EnvMismatch), +} + +#[cfg(not(windows))] +fn env_name_starts_with(name: &str, prefix: &str) -> bool { + name.starts_with(prefix) +} + +#[cfg(windows)] +fn env_name_starts_with(name: &str, prefix: &str) -> bool { + let mut name_chars = name.chars(); + for prefix_char in prefix.chars() { + let Some(name_char) = name_chars.next() else { + return false; + }; + if !name_char.eq_ignore_ascii_case(&prefix_char) { + return false; + } + } + true +} + +/// Find the first deterministic difference between stored and current env +/// glob match-sets. +fn first_env_glob_mismatch( + stored: &BTreeMap, + current: &BTreeMap, +) -> Option { + let mut stored_iter = stored.iter(); + let mut current_iter = current.iter(); + let mut s = stored_iter.next(); + let mut c = current_iter.next(); + + loop { + match (s, c) { + (None, None) => return None, + (Some((name, _)), None) => return Some(EnvMismatch::Removed { name: name.clone() }), + (None, Some((name, _))) => return Some(EnvMismatch::Added { name: name.clone() }), + (Some((sn, sv)), Some((cn, cv))) => match sn.cmp(cn) { + std::cmp::Ordering::Equal => { + if sv != cv { + return Some(EnvMismatch::Changed { name: sn.clone() }); + } + s = stored_iter.next(); + c = current_iter.next(); + } + std::cmp::Ordering::Less => return Some(EnvMismatch::Removed { name: sn.clone() }), + std::cmp::Ordering::Greater => { + return Some(EnvMismatch::Added { name: cn.clone() }); + } + }, + } + } +} + +#[cfg(test)] +mod tests { + use std::ffi::{OsStr, OsString}; + + use super::*; + + #[cfg(unix)] + fn non_utf8_os_string() -> OsString { + use std::os::unix::ffi::OsStringExt; + + OsString::from_vec(vec![0xFF]) + } + + #[cfg(windows)] + fn non_utf8_os_string() -> OsString { + use std::os::windows::ffi::OsStringExt; + + OsString::from_wide(&[0xD800]) + } + + #[test] + fn validate_errors_on_current_non_utf8_tracked_env_value() { + let mut tracked_envs = BTreeMap::new(); + tracked_envs.insert(Str::from("PROBE_ENV"), None); + let fingerprints = + TrackedEnvFingerprints { tracked_envs, ..TrackedEnvFingerprints::default() }; + + let mut unfiltered_envs = FxHashMap::default(); + unfiltered_envs.insert( + Arc::::from(OsStr::new("PROBE_ENV")), + Arc::::from(non_utf8_os_string()), + ); + + let err = fingerprints + .validate_envs(&unfiltered_envs) + .expect_err("non-UTF-8 tracked env values must error"); + + assert!(err.to_string().contains("tracked env value for PROBE_ENV is not valid UTF-8")); + } + + #[test] + fn validate_reports_current_non_utf8_tracked_env_glob_value_as_changed() { + let mut tracked_env_queries = BTreeMap::new(); + tracked_env_queries.insert(TrackedEnvQuery::Glob(Str::from("PROBE_*")), BTreeMap::new()); + let fingerprints = + TrackedEnvFingerprints { tracked_env_queries, ..TrackedEnvFingerprints::default() }; + + let mut unfiltered_envs = FxHashMap::default(); + unfiltered_envs.insert( + Arc::::from(OsStr::new("PROBE_BAD")), + Arc::::from(non_utf8_os_string()), + ); + + let mismatch = fingerprints.validate_envs(&unfiltered_envs).expect("validation succeeds"); + + match mismatch { + Some(PostRunMismatch::TrackedEnvQuery { + query, + mismatch: EnvMismatch::Changed { name }, + }) => { + assert_eq!(query, TrackedEnvQuery::Glob(Str::from("PROBE_*"))); + assert_eq!(name.as_str(), "PROBE_BAD"); + } + other => panic!("expected changed tracked env query mismatch, got {other:?}"), + } + } + + #[test] + fn validate_tracked_env_prefix_treats_star_literally() { + let mut tracked_env_queries = BTreeMap::new(); + let mut stored_matches = BTreeMap::new(); + stored_matches.insert(Str::from("PROBE_*A"), EnvValueHash::new("literal")); + tracked_env_queries.insert(TrackedEnvQuery::Prefix(Str::from("PROBE_*")), stored_matches); + let fingerprints = + TrackedEnvFingerprints { tracked_env_queries, ..TrackedEnvFingerprints::default() }; + + let mut unfiltered_envs = FxHashMap::default(); + unfiltered_envs.insert( + Arc::::from(OsStr::new("PROBE_*A")), + Arc::::from(OsStr::new("literal")), + ); + unfiltered_envs.insert( + Arc::::from(OsStr::new("PROBE_XA")), + Arc::::from(OsStr::new("wildcard if interpreted as glob")), + ); + + let mismatch = fingerprints.validate_envs(&unfiltered_envs).expect("validation succeeds"); + + assert!(mismatch.is_none()); + } + + #[test] + fn validate_ignores_non_utf8_tracked_env_glob_names() { + let mut tracked_env_queries = BTreeMap::new(); + tracked_env_queries.insert(TrackedEnvQuery::Glob(Str::from("PROBE_*")), BTreeMap::new()); + let fingerprints = + TrackedEnvFingerprints { tracked_env_queries, ..TrackedEnvFingerprints::default() }; + + let mut unfiltered_envs = FxHashMap::default(); + unfiltered_envs.insert( + Arc::::from(non_utf8_os_string()), + Arc::::from(OsStr::new("value")), + ); + + let mismatch = fingerprints.validate_envs(&unfiltered_envs).expect("validation succeeds"); + + assert!(mismatch.is_none()); + } +} diff --git a/crates/vt/src/session/reporter/summary.rs b/crates/vt/src/session/reporter/summary.rs index c9da7dff2..2d2713498 100644 --- a/crates/vt/src/session/reporter/summary.rs +++ b/crates/vt/src/session/reporter/summary.rs @@ -24,7 +24,7 @@ use crate::session::{ CacheDisabledReason, CacheErrorKind, CacheNotUpdatedReason, CacheStatus, CacheUpdateStatus, ExecutionError, }, - execute::fingerprint::TrackedEnvQuery, + execute::post_run::TrackedEnvQuery, }; // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━