forked from cobalt-org/cobalt.rs
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
RFC feat: Increase control over files to process
The `ignore` field in `.cobalt.yml` is changing from a list of glob patterns to a list `gitignore` patterns, allowing whitelisting and more advanced whitelisting/blacklisting with well defined semantics. Previuously, Cobalt processed all files except `.dot` and `_hidden`, *dest* files/directories. Considerations when designing this: - User workflow: users need `.dot` asset files, like `.htaccess`, copied over. - Implementation: Finding files or pages and posts is coupled together which will be a problem as we refactor them into more general collections or sections. - Performance: Eventually we'll want to implement incremental rebuild (see issue cobalt-org#81) and to do so we'll need to know what collection a file notification is relevant to. `gitignore` patterns were selected for the first item to ensure we had well defined semantics for how blacklist and whitelist features would interact, directory vs file patterns, etc. The last two are the motivation for factoring out file detection. Debugging impact: Ideally we'd provide a way for user's to find out why a file is whitelisted or blacklisted but that is being left for a future change. Fixes cobalt-org#221 by allowing the user to whitelist `.dot` and `_hidden` files. See the added unit tests for examples. BREAKING CHANGE: The format of `.cobalt.yml`'s `ignore` field has changed from a glob pattern to use gitignore patterns.
- Loading branch information
Showing
15 changed files
with
314 additions
and
9 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,264 @@ | ||
use std::path::{Path, PathBuf}; | ||
|
||
use ignore::Match; | ||
use ignore::gitignore::{Gitignore, GitignoreBuilder}; | ||
use walkdir::{WalkDir, DirEntry, WalkDirIterator}; | ||
|
||
use error::Result; | ||
|
||
pub struct FilesBuilder { | ||
root_dir: PathBuf, | ||
ignore: GitignoreBuilder, | ||
} | ||
|
||
impl FilesBuilder { | ||
pub fn new(root_dir: &Path) -> Result<FilesBuilder> { | ||
let mut ignore = GitignoreBuilder::new(root_dir); | ||
ignore.add_line(None, "**/.*")?; | ||
ignore.add_line(None, "**/.*/**")?; | ||
ignore.add_line(None, "**/_*")?; | ||
ignore.add_line(None, "**/_*/**")?; | ||
let builder = FilesBuilder { | ||
root_dir: root_dir.to_path_buf(), | ||
ignore: ignore, | ||
}; | ||
|
||
Ok(builder) | ||
} | ||
|
||
pub fn add_ignore(&mut self, line: &str) -> Result<()> { | ||
self.ignore.add_line(None, line)?; | ||
Ok(()) | ||
} | ||
|
||
pub fn build(&self) -> Result<Files> { | ||
let files = Files::new(self.root_dir.as_path(), self.ignore.build()?); | ||
Ok(files) | ||
} | ||
} | ||
|
||
pub struct FilesIterator<'a> { | ||
inner: Box<Iterator<Item = PathBuf> + 'a>, | ||
} | ||
|
||
impl<'a> FilesIterator<'a> { | ||
fn new(files: &'a Files) -> FilesIterator<'a> { | ||
let walker = WalkDir::new(files.root_dir.as_path()) | ||
.min_depth(1) | ||
.follow_links(false) | ||
.into_iter() | ||
.filter_entry(move |e| files.ignore_filter(e)) | ||
.filter_map(|e| e.ok()) | ||
.filter(|e| e.file_type().is_file()) | ||
.filter_map(move |e| { | ||
e.path() | ||
.strip_prefix(files.root_dir.as_path()) | ||
.ok() | ||
.map(|p| p.to_path_buf()) | ||
}); | ||
FilesIterator { inner: Box::new(walker) } | ||
} | ||
} | ||
|
||
impl<'a> Iterator for FilesIterator<'a> { | ||
type Item = PathBuf; | ||
|
||
fn next(&mut self) -> Option<PathBuf> { | ||
self.inner.next() | ||
} | ||
} | ||
|
||
pub struct Files { | ||
root_dir: PathBuf, | ||
ignore: Gitignore, | ||
} | ||
|
||
impl Files { | ||
fn new(root_dir: &Path, ignore: Gitignore) -> Files { | ||
Files { | ||
root_dir: root_dir.to_path_buf(), | ||
ignore: ignore, | ||
} | ||
} | ||
|
||
pub fn includes_file(&self, file: &Path) -> bool { | ||
let is_dir = false; | ||
match self.ignore.matched(file, is_dir) { | ||
Match::None => true, | ||
Match::Ignore(_) => false, | ||
Match::Whitelist(_) => true, | ||
} | ||
} | ||
|
||
pub fn includes_dir(&self, dir: &Path) -> bool { | ||
let is_dir = true; | ||
match self.ignore.matched(dir, is_dir) { | ||
Match::None => true, | ||
Match::Ignore(_) => false, | ||
Match::Whitelist(_) => true, | ||
} | ||
} | ||
|
||
pub fn files<'a>(&'a self) -> FilesIterator<'a> { | ||
FilesIterator::new(self) | ||
} | ||
|
||
fn ignore_filter(&self, entry: &DirEntry) -> bool { | ||
match self.ignore | ||
.matched(entry.path(), entry.file_type().is_dir()) { | ||
Match::None => true, | ||
Match::Ignore(_) => false, | ||
Match::Whitelist(_) => true, | ||
} | ||
} | ||
} | ||
|
||
#[cfg(test)] | ||
mod tests { | ||
use super::*; | ||
|
||
#[test] | ||
fn files_includes_root_dir() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(files.includes_dir(Path::new("/usr/cobalt/site"))); | ||
} | ||
|
||
#[test] | ||
fn files_includes_child_dir() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(files.includes_dir(Path::new("/usr/cobalt/site/child"))); | ||
} | ||
|
||
#[test] | ||
fn files_excludes_hidden_dir() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_dir(Path::new("/usr/cobalt/site/_child"))); | ||
|
||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_dir(Path::new("/usr/cobalt/site/child/_child"))); | ||
|
||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_dir(Path::new("/usr/cobalt/site/_child/child"))); | ||
} | ||
|
||
#[test] | ||
fn files_excludes_dot_dir() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_dir(Path::new("/usr/cobalt/site/.child"))); | ||
|
||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_dir(Path::new("/usr/cobalt/site/.child/child"))); | ||
} | ||
|
||
#[test] | ||
fn files_includes_file() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(files.includes_file(Path::new("/usr/cobalt/site/child.txt"))); | ||
} | ||
|
||
#[test] | ||
fn files_includes_child_dir_file() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(files.includes_file(Path::new("/usr/cobalt/site/child/child.txt"))); | ||
} | ||
|
||
#[test] | ||
fn files_excludes_hidden_file() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/_child.txt"))); | ||
|
||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/child/_child.txt"))); | ||
} | ||
|
||
#[test] | ||
fn files_excludes_hidden_dir_file() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/_child/child.txt"))); | ||
|
||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/child/_child/child.txt"))); | ||
} | ||
|
||
#[test] | ||
fn files_excludes_dot_file() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/.child.txt"))); | ||
|
||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/child/.child.txt"))); | ||
} | ||
|
||
#[test] | ||
fn files_excludes_dot_dir_file() { | ||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/.child/child.txt"))); | ||
|
||
let files = FilesBuilder::new(Path::new("/user/cobalt/site")) | ||
.unwrap() | ||
.build() | ||
.unwrap(); | ||
assert!(!files.includes_file(Path::new("/usr/cobalt/site/child/.child/child.txt"))); | ||
} | ||
|
||
#[test] | ||
fn files_iter_matches_include() { | ||
let root_dir = Path::new("tests/fixtures/hidden_files"); | ||
let files = FilesBuilder::new(root_dir).unwrap().build().unwrap(); | ||
let mut actual: Vec<_> = files.files().collect(); | ||
actual.sort(); | ||
|
||
let expected = vec![Path::new("child/child.txt").to_path_buf(), | ||
Path::new("child.txt").to_path_buf()]; | ||
|
||
assert_eq!(expected, actual); | ||
} | ||
} |
Oops, something went wrong.