RFC feat: Increase control over files to process

The `ignore` field in `.cobalt.yml` is changing from a list of glob patterns to
a list `gitignore` patterns, allowing whitelisting and more advanced
whitelisting/blacklisting with well defined semantics.

Previuously, Cobalt processed all files except `.dot` and `_hidden`,
*dest* files/directories.

Considerations when designing this:
- User workflow: users need `.dot` asset files, like `.htaccess`, copied over.
- Implementation: Finding files or pages and posts is coupled together
  which will be a problem as we refactor them into more general
  collections or sections.
- Performance: Eventually we'll want to implement incremental rebuild
  (see issue cobalt-org#81) and to do so we'll need to know what collection a file
  notification is relevant to.

`gitignore` patterns were selected for the first item to ensure we had
well defined semantics for how blacklist and whitelist features would
interact, directory vs file patterns, etc.

The last two are the motivation for factoring out file detection.

Debugging impact: Ideally we'd provide a way for user's to find out why a
file is whitelisted or blacklisted but that is being left for a future
change.

Fixes cobalt-org#221 by allowing the user to whitelist `.dot` and `_hidden` files.
See the added unit tests for examples.

BREAKING CHANGE: The format of `.cobalt.yml`'s `ignore` field has changed
from a glob pattern to use gitignore patterns.

Cargo.toml

@@ -39,6 +39,7 @@ clippy = {version = "0.0", optional = true}
 error-chain = "0.10.0"
 lazy_static = "0.2"
 itertools = "0.5.9"
+ignore = "0.2.0"
 
 [dependencies.hyper]
 version = "0.10"

src/error.rs

@@ -3,6 +3,7 @@ use std::io;
 use yaml_rust::scanner;
 use walkdir;
 use liquid;
+use ignore;
 
 error_chain! {
 
@@ -14,6 +15,7 @@ error_chain! {
         Liquid(liquid::Error);
         WalkDir(walkdir::Error);
         Yaml(scanner::ScanError);
+        Ignore(ignore::Error);
     }
 
     errors {

src/files.rs

@@ -0,0 +1,238 @@
+use std::path::{Path, PathBuf};
+
+use ignore::Match;
+use ignore::gitignore::{Gitignore, GitignoreBuilder};
+use walkdir::{WalkDir, DirEntry, WalkDirIterator, IterFilterEntry};
+
+use error::Result;
+
+pub struct FilesBuilder {
+    root_dir: PathBuf,
+    ignore: GitignoreBuilder,
+}
+
+impl FilesBuilder {
+    pub fn new(root_dir: &Path) -> Result<FilesBuilder> {
+        let mut ignore = GitignoreBuilder::new(root_dir);
+        ignore.add_line(None, "**/.*")?;
+        ignore.add_line(None, "**/.*/**")?;
+        ignore.add_line(None, "**/_*")?;
+        ignore.add_line(None, "**/_*/**")?;
+        let builder = FilesBuilder {
+            root_dir: root_dir.to_path_buf(),
+            ignore: ignore,
+        };
+
+        Ok(builder)
+    }
+
+    pub fn add_ignore(&mut self, line: &str) -> Result<()> {
+        self.ignore.add_line(None, line)?;
+        Ok(())
+    }
+
+    pub fn build(&self) -> Result<Files> {
+        let files = Files::new(self.root_dir.as_path(), self.ignore.build()?);
+        Ok(files)
+    }
+}
+
+pub struct FilesIterator<'a> {
+    inner: Box<Iterator<Item = PathBuf> + 'a>,
+}
+
+impl<'a> FilesIterator<'a> {
+    fn new(files: &'a Files) -> FilesIterator<'a> {
+        let walker = WalkDir::new(files.root_dir.as_path())
+            .min_depth(1)
+            .follow_links(false)
+            .into_iter()
+            .filter_entry(move |e| files.ignore_filter(e))
+            .filter_map(|e| e.ok())
+            .filter_map(move |e| {
+                            e.path()
+                                .strip_prefix(files.root_dir.as_path())
+                                .ok()
+                                .map(|p| p.to_path_buf())
+                        });
+        FilesIterator { inner: Box::new(walker) }
+    }
+}
+
+impl<'a> Iterator for FilesIterator<'a> {
+    type Item = PathBuf;
+
+    fn next(&mut self) -> Option<PathBuf> {
+        self.inner.next()
+    }
+}
+
+pub struct Files {
+    root_dir: PathBuf,
+    ignore: Gitignore,
+}
+
+impl Files {
+    fn new(root_dir: &Path, ignore: Gitignore) -> Files {
+        Files {
+            root_dir: root_dir.to_path_buf(),
+            ignore: ignore,
+        }
+    }
+
+    pub fn includes_file(&self, file: &Path) -> bool {
+        let is_dir = false;
+        match self.ignore.matched(file, is_dir) {
+            Match::None => true,
+            Match::Ignore(_) => false,
+            Match::Whitelist(_) => true,
+        }
+    }
+
+    pub fn includes_dir(&self, dir: &Path) -> bool {
+        let is_dir = true;
+        match self.ignore.matched(dir, is_dir) {
+            Match::None => true,
+            Match::Ignore(_) => false,
+            Match::Whitelist(_) => true,
+        }
+    }
+
+    pub fn files<'a>(&'a self) -> FilesIterator<'a> {
+        FilesIterator::new(self)
+    }
+
+    fn ignore_filter(&self, entry: &DirEntry) -> bool {
+        match self.ignore
+                  .matched(entry.path(), entry.file_type().is_dir()) {
+            Match::None => false,
+            Match::Ignore(_) => true,
+            Match::Whitelist(_) => false,
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn files_includes_root_dir() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(files.includes_dir(Path::new("/usr/cobalt/site")));
+    }
+
+    #[test]
+    fn files_includes_child_dir() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(files.includes_dir(Path::new("/usr/cobalt/site/child")));
+    }
+
+    #[test]
+    fn files_excludes_hidden_dir() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_dir(Path::new("/usr/cobalt/site/_child")));
+
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_dir(Path::new("/usr/cobalt/site/child/_child")));
+
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_dir(Path::new("/usr/cobalt/site/_child/child")));
+    }
+
+    #[test]
+    fn files_excludes_dot_dir() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_dir(Path::new("/usr/cobalt/site/.child")));
+
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_dir(Path::new("/usr/cobalt/site/.child/child")));
+    }
+
+    #[test]
+    fn files_includes_file() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(files.includes_file(Path::new("/usr/cobalt/site/child.txt")));
+    }
+
+    #[test]
+    fn files_includes_child_dir_file() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(files.includes_file(Path::new("/usr/cobalt/site/child/child.txt")));
+    }
+
+    #[test]
+    fn files_excludes_hidden_file() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_file(Path::new("/usr/cobalt/site/_child.txt")));
+    }
+
+    #[test]
+    fn files_excludes_hidden_dir_file() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_file(Path::new("/usr/cobalt/site/_child/child.txt")));
+
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_file(Path::new("/usr/cobalt/site/child/_child/child.txt")));
+    }
+
+    #[test]
+    fn files_excludes_dot_file() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_file(Path::new("/usr/cobalt/site/.child.txt")));
+    }
+
+    #[test]
+    fn files_excludes_dot_dir_file() {
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_file(Path::new("/usr/cobalt/site/.child/child.txt")));
+
+        let files = FilesBuilder::new(Path::new("/user/cobalt/site"))
+            .unwrap()
+            .build()
+            .unwrap();
+        assert!(!files.includes_file(Path::new("/usr/cobalt/site/child/.child/child.txt")));
+    }
+}

src/lib.rs

@@ -9,14 +9,15 @@
         too_many_arguments,
         ))]
 
+extern crate chrono;
+extern crate glob;
+extern crate ignore;
 extern crate liquid;
 extern crate pulldown_cmark;
+extern crate regex;
+extern crate rss;
 extern crate walkdir;
-extern crate chrono;
 extern crate yaml_rust;
-extern crate rss;
-extern crate glob;
-extern crate regex;
 
 extern crate itertools;
 
@@ -29,20 +30,21 @@ extern crate log;
 #[macro_use]
 extern crate error_chain;
 
+#[macro_use]
+extern crate lazy_static;
+
 pub use cobalt::build;
 pub use error::Error;
 pub use config::Config;
 pub use new::create_new_project;
 
-// modules
+pub mod error;
+
 mod cobalt;
 mod config;
-pub mod error;
 mod document;
 mod new;
+mod files;
 
 #[cfg(feature="syntax-highlight")]
 mod syntax_highlight;
-
-#[macro_use]
-extern crate lazy_static;