From a7c8cf3cdd6595370315021ace7770f85a647477 Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Tue, 8 Aug 2023 11:19:28 -0400 Subject: [PATCH 1/3] feat(copy): add more copying APIs to LocalAsset * copy_named allows you to specify a name for the dest file * copy_dir is like copy but for dirs * copy_dir_named is both put together These APIs are desired for cargo-dist --- src/error.rs | 2 +- src/local.rs | 94 ++++++++++++++++++++++++++++++++- tests/local_copy.rs | 125 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 210 insertions(+), 11 deletions(-) diff --git a/src/error.rs b/src/error.rs index 29c7c5a..3543d46 100644 --- a/src/error.rs +++ b/src/error.rs @@ -188,7 +188,7 @@ pub enum AxoassetError { dest_path: String, /// Details of the error #[source] - details: std::io::Error, + details: Option, }, /// This error indicates that axoasset failed to read a local asset at the diff --git a/src/local.rs b/src/local.rs index ca064e4..aed2d8a 100644 --- a/src/local.rs +++ b/src/local.rs @@ -212,7 +212,10 @@ impl LocalAsset { Ok(()) } - /// Copies an asset from one location on the local filesystem to another + /// Copies an asset from one location on the local filesystem to the given directory + /// + /// The destination will use the same file name as the origin has. + /// If you want to specify the destination file's name, use [`LocalAsset::copy_named`][]. pub fn copy( origin_path: impl AsRef, dest_dir: impl AsRef, @@ -220,6 +223,95 @@ impl LocalAsset { LocalAsset::load(origin_path)?.write(dest_dir) } + /// Copies an asset from one location on the local filesystem to another + /// + /// Both paths are assumed to be file names. + pub fn copy_named( + origin_path: impl AsRef, + dest_path: impl AsRef, + ) -> Result<()> { + let origin_path = origin_path.as_ref(); + let dest_path = dest_path.as_ref(); + + fs::copy(origin_path, dest_path).map_err(|e| AxoassetError::LocalAssetCopyFailed { + origin_path: origin_path.to_string(), + dest_path: dest_path.to_string(), + details: Some(e), + })?; + + Ok(()) + } + + /// Recursively copies a directory from one location to the given directory + /// + /// The destination will use the same dir name as the origin has, so + /// dest_dir is the *parent* of the copied directory. If you want to specify the destination's + /// dir name, use [`LocalAsset::copy_dir_named`][]. + pub fn copy_dir( + origin_path: impl AsRef, + dest_dir: impl AsRef, + ) -> Result { + let origin_path = origin_path.as_ref(); + let dest_dir = dest_dir.as_ref(); + + let filename = Self::filename(origin_path)?; + let dest_path = dest_dir.join(filename); + Self::copy_dir_named(origin_path, &dest_path)?; + Ok(dest_path) + } + + /// Recursively copies a directory from one location to another + /// + /// Both paths are assumed to be the names of the directory being copied + /// (i.e. dest_path is not the parent dir). + pub fn copy_dir_named( + origin_path: impl AsRef, + dest_path: impl AsRef, + ) -> Result<()> { + let origin_path = origin_path.as_ref(); + let dest_path = dest_path.as_ref(); + + for entry in walkdir::WalkDir::new(origin_path) { + let entry = entry.map_err(|e| AxoassetError::LocalAssetCopyFailed { + origin_path: origin_path.to_string(), + dest_path: dest_path.to_string(), + details: e.into_io_error(), + })?; + + let from = Utf8PathBuf::from_path_buf(entry.path().to_owned()) + .map_err(|details| AxoassetError::Utf8Path { path: details })?; + let suffix = from.strip_prefix(origin_path).map_err(|_| { + AxoassetError::LocalAssetCopyFailed { + origin_path: origin_path.to_string(), + dest_path: dest_path.to_string(), + details: None, + } + })?; + let to = dest_path.join(suffix); + + if entry.file_type().is_dir() { + // create directories (even empty ones!) + if let Err(e) = fs::create_dir(to) { + match e.kind() { + std::io::ErrorKind::AlreadyExists => {} + _ => { + return Err(AxoassetError::LocalAssetCopyFailed { + origin_path: origin_path.to_string(), + dest_path: dest_path.to_string(), + details: Some(e), + }) + } + } + } + } else if entry.file_type().is_file() { + // copy files + LocalAsset::copy_named(from, to)?; + } + // other kinds of file presumed to be symlinks which we don't handle + } + Ok(()) + } + /// Get the current working directory pub fn current_dir() -> Result { let cur_dir = diff --git a/tests/local_copy.rs b/tests/local_copy.rs index a598511..15c4caa 100644 --- a/tests/local_copy.rs +++ b/tests/local_copy.rs @@ -1,15 +1,15 @@ #![allow(irrefutable_let_patterns)] use std::collections::HashMap; -use std::path::Path; use assert_fs::prelude::*; +use camino::Utf8Path; #[tokio::test] async fn it_copies_local_assets() { let origin = assert_fs::TempDir::new().unwrap(); let dest = assert_fs::TempDir::new().unwrap(); - let dest_dir = Path::new(dest.to_str().unwrap()); + let dest_dir = Utf8Path::from_path(dest.path()).unwrap(); let mut files = HashMap::new(); files.insert("README.md", "# axoasset"); @@ -17,19 +17,15 @@ async fn it_copies_local_assets() { for (file, contents) in files { let asset = origin.child(file); - let content = Path::new("./tests/assets").join(file); - asset.write_file(&content).unwrap(); + asset.write_str(contents).unwrap(); - let origin_path = asset.to_str().unwrap(); - axoasset::Asset::copy(origin_path, dest.to_str().unwrap()) + axoasset::Asset::copy(asset.to_str().unwrap(), dest.to_str().unwrap()) .await .unwrap(); let copied_file = dest_dir.join(file); assert!(copied_file.exists()); - let loaded_asset = axoasset::Asset::load(copied_file.to_str().unwrap()) - .await - .unwrap(); + let loaded_asset = axoasset::Asset::load(copied_file.as_str()).await.unwrap(); if let axoasset::Asset::LocalAsset(asset) = loaded_asset { assert!(std::str::from_utf8(&asset.contents) .unwrap() @@ -37,3 +33,114 @@ async fn it_copies_local_assets() { } } } + +#[tokio::test] +async fn it_copies_named_local_assets() { + let origin = assert_fs::TempDir::new().unwrap(); + let dest = assert_fs::TempDir::new().unwrap(); + let dest_dir = Utf8Path::from_path(dest.path()).unwrap(); + + let mut files = HashMap::new(); + files.insert("README.md", "# axoasset"); + files.insert("styles.css", "@import"); + + for (file, contents) in files { + let asset = origin.child(file); + asset.write_str(contents).unwrap(); + + let origin_path = asset.to_str().unwrap(); + axoasset::LocalAsset::copy_named(origin_path, dest_dir.join(file)).unwrap(); + + let copied_file = dest_dir.join(file); + assert!(copied_file.exists()); + let loaded_asset = axoasset::LocalAsset::load(copied_file).unwrap(); + assert!(std::str::from_utf8(&loaded_asset.contents) + .unwrap() + .contains(contents)); + } +} + +#[tokio::test] +async fn it_copies_dirs() { + let origin = assert_fs::TempDir::new().unwrap().child("result"); + let dest = assert_fs::TempDir::new().unwrap(); + let origin_dir = Utf8Path::from_path(origin.path()).unwrap(); + let dest_dir = Utf8Path::from_path(dest.path()).unwrap(); + origin.create_dir_all().unwrap(); + + // None means it's just a dir, used to make sure empty dirs get copied + let mut files = HashMap::new(); + files.insert("blah/blargh/README3.md", Some("# axoasset3")); + files.insert("blah/README2.md", Some("# axoasset2")); + files.insert("blah/README.md", Some("# axoasset")); + files.insert("styles.css", Some("@import")); + files.insert("blah/blargh/empty_dir", None); + files.insert("empty/dirs", None); + files.insert("root_empty", None); + + for (file, contents) in &files { + let asset = origin.child(file); + if let Some(contents) = contents { + std::fs::create_dir_all(asset.parent().unwrap()).unwrap(); + asset.write_str(contents).unwrap(); + } else { + asset.create_dir_all().unwrap(); + } + } + + axoasset::LocalAsset::copy_dir(origin_dir, dest_dir).unwrap(); + + for (file, contents) in &files { + let copied_file = dest_dir.join("result").join(file); + + assert!(copied_file.exists()); + if let Some(contents) = contents { + let loaded_asset = axoasset::LocalAsset::load(copied_file).unwrap(); + assert!(std::str::from_utf8(&loaded_asset.contents) + .unwrap() + .contains(contents)); + } + } +} + +#[tokio::test] +async fn it_copies_named_dirs() { + let origin = assert_fs::TempDir::new().unwrap(); + let dest = assert_fs::TempDir::new().unwrap(); + let origin_dir = Utf8Path::from_path(origin.path()).unwrap(); + let dest_dir = Utf8Path::from_path(dest.path()).unwrap().join("result"); + + // None means it's just a dir, used to make sure empty dirs get copied + let mut files = HashMap::new(); + files.insert("blah/blargh/README3.md", Some("# axoasset3")); + files.insert("blah/README2.md", Some("# axoasset2")); + files.insert("blah/README.md", Some("# axoasset")); + files.insert("styles.css", Some("@import")); + files.insert("blah/blargh/empty_dir", None); + files.insert("empty/dirs", None); + files.insert("root_empty", None); + + for (file, contents) in &files { + let asset = origin.child(file); + if let Some(contents) = contents { + std::fs::create_dir_all(asset.parent().unwrap()).unwrap(); + asset.write_str(contents).unwrap(); + } else { + asset.create_dir_all().unwrap(); + } + } + + axoasset::LocalAsset::copy_dir_named(origin_dir, &dest_dir).unwrap(); + + for (file, contents) in &files { + let copied_file = dest_dir.join(file); + + assert!(copied_file.exists()); + if let Some(contents) = contents { + let loaded_asset = axoasset::LocalAsset::load(copied_file).unwrap(); + assert!(std::str::from_utf8(&loaded_asset.contents) + .unwrap() + .contains(contents)); + } + } +} From 22853664d6d97df554116ebfe1d7b37fb95140cb Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Tue, 8 Aug 2023 14:14:00 -0400 Subject: [PATCH 2/3] feat(copy): factor out a WalkDirs wrapper and clean up copy code --- src/compression.rs | 15 +++----- src/dirs.rs | 89 ++++++++++++++++++++++++++++++++++++++++++++++ src/error.rs | 22 +++++++++++- src/lib.rs | 4 ++- src/local.rs | 54 +++++++++++----------------- src/remote.rs | 8 +++-- 6 files changed, 143 insertions(+), 49 deletions(-) create mode 100644 src/dirs.rs diff --git a/src/compression.rs b/src/compression.rs index d6cc782..adcbe32 100644 --- a/src/compression.rs +++ b/src/compression.rs @@ -166,14 +166,14 @@ pub(crate) fn zip_dir( fs::File, io::{Read, Write}, }; - use zip::{result::ZipError, write::FileOptions, CompressionMethod}; + use zip::{write::FileOptions, CompressionMethod}; let file = File::create(dest_path)?; // The `zip` crate lacks the conveniences of the `tar` crate so we need to manually // walk through all the subdirs of `src_path` and copy each entry. walkdir streamlines // that process for us. - let walkdir = walkdir::WalkDir::new(src_path); + let walkdir = crate::dirs::walk_dir(src_path); let it = walkdir.into_iter(); let mut zip = zip::ZipWriter::new(file); @@ -190,15 +190,8 @@ pub(crate) fn zip_dir( let mut buffer = Vec::new(); for entry in it.filter_map(|e| e.ok()) { - let path = entry.path(); - // Get the relative path of this file/dir that will be used in the zip - let Some(name) = path - .strip_prefix(src_path) - .ok() - .and_then(Utf8Path::from_path) - else { - return Err(ZipError::UnsupportedArchive("unsupported path format")); - }; + let name = &entry.rel_path; + let path = &entry.full_path; // Optionally apply the root prefix let name = if let Some(root) = with_root { root.join(name) diff --git a/src/dirs.rs b/src/dirs.rs new file mode 100644 index 0000000..4957ca5 --- /dev/null +++ b/src/dirs.rs @@ -0,0 +1,89 @@ +//! Utilities for working with directories +//! +//! Right now just a wrapper around WalkDirs that does some utf8 conversions and strip_prefixing, +//! since we always end up doing that. + +use crate::error::*; +use camino::{Utf8Path, Utf8PathBuf}; + +/// Walk through this dir's descendants with `walkdirs` +pub fn walk_dir(dir: impl AsRef) -> AxoassetWalkDir { + let dir = dir.as_ref(); + AxoassetWalkDir { + root_dir: dir.to_owned(), + inner: walkdir::WalkDir::new(dir), + } +} + +/// Wrapper around [`walkdir::WalkDir`][]. +pub struct AxoassetWalkDir { + root_dir: Utf8PathBuf, + inner: walkdir::WalkDir, +} + +/// Wrapper around [`walkdir::IntoIter`][]. +pub struct AxoassetIntoIter { + root_dir: Utf8PathBuf, + inner: walkdir::IntoIter, +} + +/// Wrapper around [`walkdir::DirEntry`][]. +pub struct AxoassetDirEntry { + /// full path to the entry + pub full_path: Utf8PathBuf, + /// path to the entry relative to the dir passed to [`walk_dir`][]. + pub rel_path: Utf8PathBuf, + /// Inner contents + pub entry: walkdir::DirEntry, +} + +impl IntoIterator for AxoassetWalkDir { + type IntoIter = AxoassetIntoIter; + type Item = Result; + fn into_iter(self) -> Self::IntoIter { + AxoassetIntoIter { + root_dir: self.root_dir, + inner: self.inner.into_iter(), + } + } +} + +impl Iterator for AxoassetIntoIter { + type Item = Result; + fn next(&mut self) -> Option { + self.inner.next().map(|next| { + let entry = next.map_err(|e| AxoassetError::WalkDirFailed { + origin_path: self.root_dir.clone(), + details: e, + })?; + + let full_path = Utf8PathBuf::from_path_buf(entry.path().to_owned()) + .map_err(|details| AxoassetError::Utf8Path { path: details })?; + let rel_path = full_path + .strip_prefix(&self.root_dir) + .map_err(|_| AxoassetError::PathNesting { + root_dir: self.root_dir.clone(), + child_dir: full_path.clone(), + })? + .to_owned(); + + Ok(AxoassetDirEntry { + full_path, + rel_path, + entry, + }) + }) + } +} + +impl std::ops::Deref for AxoassetDirEntry { + type Target = walkdir::DirEntry; + fn deref(&self) -> &Self::Target { + &self.entry + } +} +impl std::ops::DerefMut for AxoassetDirEntry { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.entry + } +} diff --git a/src/error.rs b/src/error.rs index 3543d46..8094024 100644 --- a/src/error.rs +++ b/src/error.rs @@ -188,7 +188,7 @@ pub enum AxoassetError { dest_path: String, /// Details of the error #[source] - details: Option, + details: std::io::Error, }, /// This error indicates that axoasset failed to read a local asset at the @@ -292,6 +292,16 @@ pub enum AxoassetError { /// The problematic path path: std::path::PathBuf, }, + /// This error indicates we tried to strip_prefix a path that should have been + /// a descendant of another, but it didn't work. + #[error("Child wasn't nested under its parent: {root_dir} => {child_dir}")] + #[diagnostic(help("Are symlinks involved?"))] + PathNesting { + /// The root/ancestor dir + root_dir: camino::Utf8PathBuf, + /// THe child/descendent path + child_dir: camino::Utf8PathBuf, + }, #[error("Failed to find {desired_filename} in an ancestor of {start_dir}")] /// This error indicates we failed to find the desired file in an ancestor of the search dir. @@ -302,6 +312,16 @@ pub enum AxoassetError { desired_filename: String, }, + #[error("Failed to walk to ancestor of {origin_path}")] + /// Walkdir failed to yield an entry + WalkDirFailed { + /// The root path we were trying to walkdirs + origin_path: camino::Utf8PathBuf, + /// Inner walkdir error + #[source] + details: walkdir::Error, + }, + /// This error indicates we tried to deserialize some JSON with serde_json /// but failed. #[cfg(feature = "json-serde")] diff --git a/src/lib.rs b/src/lib.rs index 9ea1ade..77c53a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,6 +13,7 @@ use std::path::PathBuf; #[cfg(any(feature = "compression-zip", feature = "compression-tar"))] pub(crate) mod compression; +pub(crate) mod dirs; pub(crate) mod error; pub(crate) mod local; #[cfg(feature = "remote")] @@ -20,6 +21,7 @@ pub(crate) mod remote; pub(crate) mod source; pub(crate) mod spanned; +use camino::Utf8PathBuf; pub use error::AxoassetError; use error::Result; pub use local::LocalAsset; @@ -83,7 +85,7 @@ impl Asset { /// Copies an asset, returning the path to the copy destination on the /// local filesystem. - pub async fn copy(origin_path: &str, dest_dir: &str) -> Result { + pub async fn copy(origin_path: &str, dest_dir: &str) -> Result { #[cfg(feature = "remote")] if is_remote(origin_path)? { return RemoteAsset::copy(origin_path, dest_dir).await; diff --git a/src/local.rs b/src/local.rs index aed2d8a..0521349 100644 --- a/src/local.rs +++ b/src/local.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use camino::{Utf8Path, Utf8PathBuf}; -use crate::error::*; +use crate::{dirs, error::*}; /// A local asset contains a path on the local filesystem and its contents #[derive(Debug)] @@ -216,11 +216,20 @@ impl LocalAsset { /// /// The destination will use the same file name as the origin has. /// If you want to specify the destination file's name, use [`LocalAsset::copy_named`][]. + /// + /// The returned path is the resulting file. pub fn copy( origin_path: impl AsRef, dest_dir: impl AsRef, - ) -> Result { - LocalAsset::load(origin_path)?.write(dest_dir) + ) -> Result { + let origin_path = origin_path.as_ref(); + let dest_dir = dest_dir.as_ref(); + + let filename = Self::filename(origin_path)?; + let dest_path = dest_dir.join(filename); + Self::copy_named(origin_path, &dest_path)?; + + Ok(dest_path) } /// Copies an asset from one location on the local filesystem to another @@ -236,7 +245,7 @@ impl LocalAsset { fs::copy(origin_path, dest_path).map_err(|e| AxoassetError::LocalAssetCopyFailed { origin_path: origin_path.to_string(), dest_path: dest_path.to_string(), - details: Some(e), + details: e, })?; Ok(()) @@ -247,6 +256,8 @@ impl LocalAsset { /// The destination will use the same dir name as the origin has, so /// dest_dir is the *parent* of the copied directory. If you want to specify the destination's /// dir name, use [`LocalAsset::copy_dir_named`][]. + /// + /// The returned path is the resulting dir. pub fn copy_dir( origin_path: impl AsRef, dest_dir: impl AsRef, @@ -257,6 +268,7 @@ impl LocalAsset { let filename = Self::filename(origin_path)?; let dest_path = dest_dir.join(filename); Self::copy_dir_named(origin_path, &dest_path)?; + Ok(dest_path) } @@ -271,38 +283,14 @@ impl LocalAsset { let origin_path = origin_path.as_ref(); let dest_path = dest_path.as_ref(); - for entry in walkdir::WalkDir::new(origin_path) { - let entry = entry.map_err(|e| AxoassetError::LocalAssetCopyFailed { - origin_path: origin_path.to_string(), - dest_path: dest_path.to_string(), - details: e.into_io_error(), - })?; - - let from = Utf8PathBuf::from_path_buf(entry.path().to_owned()) - .map_err(|details| AxoassetError::Utf8Path { path: details })?; - let suffix = from.strip_prefix(origin_path).map_err(|_| { - AxoassetError::LocalAssetCopyFailed { - origin_path: origin_path.to_string(), - dest_path: dest_path.to_string(), - details: None, - } - })?; - let to = dest_path.join(suffix); + for entry in dirs::walk_dir(origin_path) { + let entry = entry?; + let from = &entry.full_path; + let to = dest_path.join(&entry.rel_path); if entry.file_type().is_dir() { // create directories (even empty ones!) - if let Err(e) = fs::create_dir(to) { - match e.kind() { - std::io::ErrorKind::AlreadyExists => {} - _ => { - return Err(AxoassetError::LocalAssetCopyFailed { - origin_path: origin_path.to_string(), - dest_path: dest_path.to_string(), - details: Some(e), - }) - } - } - } + LocalAsset::create_dir(to)?; } else if entry.file_type().is_file() { // copy files LocalAsset::copy_named(from, to)?; diff --git a/src/remote.rs b/src/remote.rs index 6141b15..84099fb 100644 --- a/src/remote.rs +++ b/src/remote.rs @@ -1,6 +1,8 @@ use std::fs; use std::path::{Path, PathBuf}; +use camino::{Utf8Path, Utf8PathBuf}; + use crate::error::*; /// A remote asset is an asset that is fetched over the network. @@ -60,15 +62,15 @@ impl RemoteAsset { } /// Copies an asset to the local filesystem. - pub async fn copy(origin_path: &str, dest_dir: &str) -> Result { + pub async fn copy(origin_path: &str, dest_dir: &str) -> Result { match RemoteAsset::load(origin_path).await { Ok(a) => { - let dest_path = Path::new(dest_dir).join(a.filename); + let dest_path = Utf8Path::new(dest_dir).join(a.filename); match fs::write(&dest_path, a.contents) { Ok(_) => Ok(dest_path), Err(details) => Err(AxoassetError::RemoteAssetWriteFailed { origin_path: origin_path.to_string(), - dest_path: dest_path.display().to_string(), + dest_path: dest_path.to_string(), details, }), } From ad86c9c9e64312836bbdef2f172bd1bb1a7f854f Mon Sep 17 00:00:00 2001 From: Aria Beingessner Date: Tue, 8 Aug 2023 15:13:09 -0400 Subject: [PATCH 3/3] chore: add debug_assert for symlinks --- src/local.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/local.rs b/src/local.rs index 0521349..6143a81 100644 --- a/src/local.rs +++ b/src/local.rs @@ -294,8 +294,13 @@ impl LocalAsset { } else if entry.file_type().is_file() { // copy files LocalAsset::copy_named(from, to)?; + } else { + // other kinds of file presumed to be symlinks which we don't handle + debug_assert!( + entry.file_type().is_symlink(), + "unknown type of file at {from}, axoasset needs to be updated to support this!" + ); } - // other kinds of file presumed to be symlinks which we don't handle } Ok(()) }