From 7b21eddb5fbe0e0fce78b94bc9d8ee91310ac149 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Tue, 23 Apr 2024 20:52:31 -0500 Subject: [PATCH 1/3] feat(dir): Add `Dir` trait with in-memory and filesystem impls --- crates/snapbox/src/data/mod.rs | 2 +- crates/snapbox/src/dir/dir.rs | 396 +++++++++++++++++++++++++++++++++ crates/snapbox/src/dir/mod.rs | 13 ++ crates/snapbox/src/dir/root.rs | 13 +- crates/snapbox/src/lib.rs | 2 + 5 files changed, 415 insertions(+), 11 deletions(-) create mode 100644 crates/snapbox/src/dir/dir.rs diff --git a/crates/snapbox/src/data/mod.rs b/crates/snapbox/src/data/mod.rs index 90326549..506cc662 100644 --- a/crates/snapbox/src/data/mod.rs +++ b/crates/snapbox/src/data/mod.rs @@ -269,7 +269,7 @@ impl Data { Self::with_inner(DataInner::JsonLines(serde_json::Value::Array(raw.into()))) } - fn error(raw: impl Into, intended: DataFormat) -> Self { + pub fn error(raw: impl Into, intended: DataFormat) -> Self { Self::with_inner(DataInner::Error(DataError { error: raw.into(), intended, diff --git a/crates/snapbox/src/dir/dir.rs b/crates/snapbox/src/dir/dir.rs new file mode 100644 index 00000000..ec67b048 --- /dev/null +++ b/crates/snapbox/src/dir/dir.rs @@ -0,0 +1,396 @@ +use super::FileType; + +/// Collection of files +#[cfg(feature = "dir")] // for documentation purposes only +pub trait Dir { + type WalkIter; + + /// Return [`DirEntry`]s with files in binary mode + fn binary_iter(&self) -> Self::WalkIter; + /// Initialize a test fixture directory `root` + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error>; +} + +impl Dir for InMemoryDir { + type WalkIter = InMemoryDirIter; + + fn binary_iter(&self) -> Self::WalkIter { + self.clone().content.into_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + self.write_to(root) + } +} + +impl Dir for std::path::Path { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + PathIter::binary_iter(self) + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + let src = super::resolve_dir(self).map_err(|e| format!("{e}: {}", self.display()))?; + for (relpath, entry) in src.as_path().binary_iter() { + let dest = root.join(relpath); + entry.write_to(&dest)?; + } + Ok(()) + } +} + +impl Dir for &'_ std::path::Path { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + (*self).binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + (*self).write_to(root) + } +} + +impl Dir for &'_ std::path::PathBuf { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + self.as_path().binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + self.as_path().write_to(root) + } +} + +impl Dir for std::path::PathBuf { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + self.as_path().binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + self.as_path().write_to(root) + } +} + +impl Dir for str { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + std::path::Path::new(self).binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + std::path::Path::new(self).write_to(root) + } +} + +impl Dir for &'_ str { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + (*self).binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + (*self).write_to(root) + } +} + +impl Dir for &'_ String { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + self.as_str().binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + self.as_str().write_to(root) + } +} + +impl Dir for String { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + self.as_str().binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + self.as_str().write_to(root) + } +} + +impl Dir for std::ffi::OsStr { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + std::path::Path::new(self).binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + std::path::Path::new(self).write_to(root) + } +} + +impl Dir for &'_ std::ffi::OsStr { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + (*self).binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + (*self).write_to(root) + } +} + +impl Dir for &'_ std::ffi::OsString { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + self.as_os_str().binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + self.as_os_str().write_to(root) + } +} + +impl Dir for std::ffi::OsString { + type WalkIter = PathIter; + + fn binary_iter(&self) -> Self::WalkIter { + self.as_os_str().binary_iter() + } + fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + self.as_os_str().write_to(root) + } +} + +#[cfg(feature = "dir")] // for documentation purposes only +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InMemoryDir { + content: std::collections::BTreeMap, +} + +impl InMemoryDir { + /// Initialize a test fixture directory `root` + pub fn write_to(&self, root: &std::path::Path) -> Result<(), crate::assert::Error> { + for (relpath, entry) in &self.content { + let dest = root.join(relpath); + entry.write_to(&dest)?; + } + Ok(()) + } +} + +pub type InMemoryDirIter = std::collections::btree_map::IntoIter; + +impl FromIterator<(P, E)> for InMemoryDir +where + P: Into, + E: Into, +{ + fn from_iter>(it: I) -> Self { + let mut content: std::collections::BTreeMap = + Default::default(); + for (mut p, e) in it.into_iter().map(|(p, e)| (p.into(), e.into())) { + if let Ok(rel) = p.strip_prefix("/") { + p = rel.to_owned(); + } + assert!(p.is_relative(), "{}", p.display()); + let mut ancestors = p.ancestors(); + ancestors.next(); // skip self + for ancestor in ancestors { + match content.entry(ancestor.to_owned()) { + std::collections::btree_map::Entry::Occupied(entry) => { + if entry.get().file_type() != FileType::File { + panic!( + "`{}` assumes `{}` is a dir but its a {:?}", + p.display(), + ancestor.display(), + entry.get().file_type() + ); + } + } + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(DirEntry::Dir); + } + } + } + match content.entry(p) { + std::collections::btree_map::Entry::Occupied(entry) => { + panic!( + "`{}` is assumed to be empty but its a {:?}", + entry.key().display(), + entry.get().file_type() + ); + } + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(e); + } + } + } + Self { content } + } +} + +/// Note: Ignores `.keep` files +#[cfg(feature = "dir")] // for documentation purposes only +pub struct PathIter { + root: std::path::PathBuf, + binary: bool, + inner: walkdir::IntoIter, +} + +impl PathIter { + fn binary_iter(root: &std::path::Path) -> Self { + let binary = true; + Self::iter_(root, binary) + } + + fn iter_(root: &std::path::Path, binary: bool) -> PathIter { + PathIter { + root: root.to_owned(), + binary, + inner: walkdir::WalkDir::new(root).into_iter(), + } + } +} + +impl Iterator for PathIter { + type Item = (std::path::PathBuf, DirEntry); + + fn next(&mut self) -> Option { + for raw in self.inner.by_ref() { + let entry = match raw { + Ok(raw) => { + if raw.file_type().is_file() + && raw.path().file_name() == Some(std::ffi::OsStr::new(".keep")) + { + crate::debug!("ignoring {}, `.keep` file", raw.path().display()); + continue; + } + + let Ok(path) = raw.path().strip_prefix(&self.root) else { + crate::debug!( + "ignoring {}, out of root {}", + raw.path().display(), + self.root.display() + ); + continue; + }; + let entry = match DirEntry::try_from_path(raw.path(), self.binary) { + Ok(entry) => entry, + Err(err) => DirEntry::error(err), + }; + (path.to_owned(), entry) + } + Err(err) => { + let Some(path) = err.path() else { + crate::debug!("ignoring error {err}"); + continue; + }; + let Ok(path) = path.strip_prefix(&self.root) else { + crate::debug!( + "ignoring {}, out of root {}", + path.display(), + self.root.display() + ); + continue; + }; + (path.to_owned(), DirEntry::error(err.to_string().into())) + } + }; + return Some(entry); + } + None + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DirEntry { + Dir, + File(crate::Data), + Symlink(std::path::PathBuf), +} + +impl DirEntry { + fn error(err: crate::assert::Error) -> Self { + DirEntry::File(crate::Data::error(err, crate::data::DataFormat::Error)) + } + + fn try_from_path(path: &std::path::Path, binary: bool) -> Result { + let metadata = path + .symlink_metadata() + .map_err(|e| format!("{e}: {}", path.display()))?; + let entry = if metadata.is_dir() { + DirEntry::Dir + } else if metadata.is_file() { + let data = if binary { + crate::Data::binary( + std::fs::read(path).map_err(|e| format!("{e}: {}", path.display()))?, + ) + } else { + crate::Data::try_read_from(path, None) + .map_err(|e| format!("{e}: {}", path.display()))? + }; + DirEntry::File(data) + } else if metadata.is_symlink() { + DirEntry::Symlink( + path.read_link() + .map_err(|e| format!("{e}: {}", path.display()))?, + ) + } else { + return Err(crate::assert::Error::new("unknown file type")); + }; + Ok(entry) + } + + pub fn write_to(&self, path: &std::path::Path) -> Result<(), crate::assert::Error> { + match self { + DirEntry::Dir => { + std::fs::create_dir_all(path).map_err(|e| format!("{e}: {}", path.display()))? + } + DirEntry::File(content) => { + std::fs::write(path, content.to_bytes()?) + .map_err(|e| format!("{e}: {}", path.display()))?; + // Avoid a mtime check race where: + // - Copy files + // - Test checks mtime + // - Test writes + // - Test checks mtime + // + // If all of this happens too close to each other, then the second mtime check will think + // nothing was written by the test. + // + // Instead of just setting 1s in the past, we'll use a reproducible mtime + filetime::set_file_mtime(path, filetime::FileTime::zero()) + .map_err(|e| format!("{e}: {}", path.display()))?; + } + DirEntry::Symlink(target) => { + symlink_to_file(path, target).map_err(|e| format!("{e}: {}", path.display()))?; + } + } + Ok(()) + } + + pub fn file_type(&self) -> FileType { + match self { + Self::Dir => FileType::Dir, + Self::File(_) => FileType::File, + Self::Symlink(_) => FileType::Symlink, + } + } +} + +impl From for DirEntry +where + D: crate::data::IntoData, +{ + fn from(data: D) -> Self { + Self::File(data.into_data()) + } +} + +#[cfg(windows)] +fn symlink_to_file(link: &std::path::Path, target: &std::path::Path) -> Result<(), std::io::Error> { + std::os::windows::fs::symlink_file(target, link) +} + +#[cfg(not(windows))] +fn symlink_to_file(link: &std::path::Path, target: &std::path::Path) -> Result<(), std::io::Error> { + std::os::unix::fs::symlink(target, link) +} diff --git a/crates/snapbox/src/dir/mod.rs b/crates/snapbox/src/dir/mod.rs index 2787bdc1..fdd5a844 100644 --- a/crates/snapbox/src/dir/mod.rs +++ b/crates/snapbox/src/dir/mod.rs @@ -1,6 +1,9 @@ //! Initialize working directories and assert on how they've changed mod diff; +#[cfg(feature = "dir")] +#[allow(clippy::module_inception)] +mod dir; mod ops; mod root; #[cfg(test)] @@ -9,6 +12,16 @@ mod tests; pub use diff::FileType; pub use diff::PathDiff; #[cfg(feature = "dir")] +pub use dir::Dir; +#[cfg(feature = "dir")] +pub use dir::DirEntry; +#[cfg(feature = "dir")] +pub use dir::InMemoryDir; +#[cfg(feature = "dir")] +pub use dir::InMemoryDirIter; +#[cfg(feature = "dir")] +pub use dir::PathIter; +#[cfg(feature = "dir")] pub use ops::copy_template; pub use ops::resolve_dir; pub use ops::strip_trailing_slash; diff --git a/crates/snapbox/src/dir/root.rs b/crates/snapbox/src/dir/root.rs index 1981a700..4ee4a6f1 100644 --- a/crates/snapbox/src/dir/root.rs +++ b/crates/snapbox/src/dir/root.rs @@ -43,21 +43,14 @@ impl DirRoot { } #[cfg(feature = "dir")] - pub fn with_template( - self, - template_root: &std::path::Path, - ) -> Result { + pub fn with_template(self, dir: impl crate::dir::Dir) -> Result { match &self.0 { DirRootInner::None | DirRootInner::Immutable(_) => { return Err("Sandboxing is disabled".into()); } DirRootInner::MutablePath(path) | DirRootInner::MutableTemp { path, .. } => { - crate::debug!( - "Initializing {} from {}", - path.display(), - template_root.display() - ); - super::copy_template(template_root, path)?; + crate::debug!("Initializing {}", path.display()); + dir.write_to(path)?; } } diff --git a/crates/snapbox/src/lib.rs b/crates/snapbox/src/lib.rs index b60d2eea..a704a2eb 100644 --- a/crates/snapbox/src/lib.rs +++ b/crates/snapbox/src/lib.rs @@ -83,6 +83,8 @@ pub use snapbox_macros::debug; /// Easier access to common traits pub mod prelude { + #[cfg(feature = "dir")] + pub use crate::dir::Dir; pub use crate::IntoData; #[cfg(feature = "json")] pub use crate::IntoJson; From 3e3077d202e0d87fbea96a2fb4679859fc352390 Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 24 Apr 2024 19:25:57 -0500 Subject: [PATCH 2/3] fix(dir)!: Remove copy_template in favor of Dir --- crates/snapbox/src/dir/mod.rs | 2 -- crates/snapbox/src/dir/ops.rs | 30 ------------------------------ 2 files changed, 32 deletions(-) diff --git a/crates/snapbox/src/dir/mod.rs b/crates/snapbox/src/dir/mod.rs index fdd5a844..3e1ae78d 100644 --- a/crates/snapbox/src/dir/mod.rs +++ b/crates/snapbox/src/dir/mod.rs @@ -21,8 +21,6 @@ pub use dir::InMemoryDir; pub use dir::InMemoryDirIter; #[cfg(feature = "dir")] pub use dir::PathIter; -#[cfg(feature = "dir")] -pub use ops::copy_template; pub use ops::resolve_dir; pub use ops::strip_trailing_slash; #[cfg(feature = "dir")] diff --git a/crates/snapbox/src/dir/ops.rs b/crates/snapbox/src/dir/ops.rs index bd5ca5a5..002388d4 100644 --- a/crates/snapbox/src/dir/ops.rs +++ b/crates/snapbox/src/dir/ops.rs @@ -34,36 +34,6 @@ impl Iterator for Walk { } } -/// Copy a template into a [`DirRoot`][super::DirRoot] -/// -/// Note: Generally you'll use [`DirRoot::with_template`][super::DirRoot::with_template] instead. -/// -/// Note: Ignores `.keep` files -#[cfg(feature = "dir")] -pub fn copy_template( - source: impl AsRef, - dest: impl AsRef, -) -> Result<(), crate::assert::Error> { - let source = source.as_ref(); - let dest = dest.as_ref(); - let source = canonicalize(source) - .map_err(|e| format!("Failed to canonicalize {}: {}", source.display(), e))?; - std::fs::create_dir_all(dest) - .map_err(|e| format!("Failed to create {}: {}", dest.display(), e))?; - let dest = canonicalize(dest) - .map_err(|e| format!("Failed to canonicalize {}: {}", dest.display(), e))?; - - for current in Walk::new(&source) { - let current = current.map_err(|e| e.to_string())?; - let rel = current.strip_prefix(&source).unwrap(); - let target = dest.join(rel); - - shallow_copy(¤t, &target)?; - } - - Ok(()) -} - /// Copy a file system entry, without recursing pub(crate) fn shallow_copy( source: &std::path::Path, From 7277b26618c4cf7f92cc1822bb5392a647d4962d Mon Sep 17 00:00:00 2001 From: Ed Page Date: Wed, 24 Apr 2024 19:26:45 -0500 Subject: [PATCH 3/3] fix(dir)!: Remove Walk in favor of Dir --- crates/snapbox/src/dir/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/snapbox/src/dir/mod.rs b/crates/snapbox/src/dir/mod.rs index 3e1ae78d..9ad09062 100644 --- a/crates/snapbox/src/dir/mod.rs +++ b/crates/snapbox/src/dir/mod.rs @@ -23,10 +23,10 @@ pub use dir::InMemoryDirIter; pub use dir::PathIter; pub use ops::resolve_dir; pub use ops::strip_trailing_slash; -#[cfg(feature = "dir")] -pub use ops::Walk; pub use root::DirRoot; #[cfg(feature = "dir")] pub(crate) use ops::canonicalize; pub(crate) use ops::shallow_copy; +#[cfg(feature = "dir")] +pub(crate) use ops::Walk;