Permalink
Browse files

Ignore files matching patterns in .stignore (fixes #7)

  • Loading branch information...
calmh committed Jan 6, 2014
1 parent 46d828e commit 986b15573a66f95beae53b10371ba50ab147124a
Showing with 131 additions and 20 deletions.
  1. +9 −0 README.md
  2. +1 −1 main.go
  3. +7 −7 model/model_test.go
  4. +53 −11 model/walk.go
  5. +61 −1 model/walk_test.go
View
@@ -186,6 +186,15 @@ $ syncthing --gui 127.0.0.1:8080
You then point your browser to the given address.
+Excluding Files
+---------------
+
+syncthing looks for files named `.stignore` while walking the
+repository. The file is expected to contain glob patterns of file names
+to ignore. Patterns are matched on file name only and apply to files in
+the same directory as the `.stignore` file and in directories lower down
+in the hierarchy.
+
License
=======
View
@@ -333,7 +333,7 @@ func connect(myID string, addr string, nodeAddrs map[string][]string, m *model.M
}
func updateLocalModel(m *model.Model) {
- files := m.Walk(!opts.NoSymlinks)
+ files := m.FilteredWalk(!opts.NoSymlinks)
m.ReplaceLocal(files)
saveIndex(m)
}
View
@@ -58,7 +58,7 @@ func init() {
func TestUpdateLocal(t *testing.T) {
m := NewModel("testdata")
- fs := m.Walk(false)
+ fs, _ := m.Walk(false)
m.ReplaceLocal(fs)
if len(m.need) > 0 {
@@ -100,7 +100,7 @@ func TestUpdateLocal(t *testing.T) {
func TestRemoteUpdateExisting(t *testing.T) {
m := NewModel("testdata")
- fs := m.Walk(false)
+ fs, _ := m.Walk(false)
m.ReplaceLocal(fs)
newFile := protocol.FileInfo{
@@ -117,7 +117,7 @@ func TestRemoteUpdateExisting(t *testing.T) {
func TestRemoteAddNew(t *testing.T) {
m := NewModel("testdata")
- fs := m.Walk(false)
+ fs, _ := m.Walk(false)
m.ReplaceLocal(fs)
newFile := protocol.FileInfo{
@@ -134,7 +134,7 @@ func TestRemoteAddNew(t *testing.T) {
func TestRemoteUpdateOld(t *testing.T) {
m := NewModel("testdata")
- fs := m.Walk(false)
+ fs, _ := m.Walk(false)
m.ReplaceLocal(fs)
oldTimeStamp := int64(1234)
@@ -152,7 +152,7 @@ func TestRemoteUpdateOld(t *testing.T) {
func TestRemoteIndexUpdate(t *testing.T) {
m := NewModel("testdata")
- fs := m.Walk(false)
+ fs, _ := m.Walk(false)
m.ReplaceLocal(fs)
foo := protocol.FileInfo{
@@ -185,7 +185,7 @@ func TestRemoteIndexUpdate(t *testing.T) {
func TestDelete(t *testing.T) {
m := NewModel("testdata")
- fs := m.Walk(false)
+ fs, _ := m.Walk(false)
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -275,7 +275,7 @@ func TestDelete(t *testing.T) {
func TestForgetNode(t *testing.T) {
m := NewModel("testdata")
- fs := m.Walk(false)
+ fs, _ := m.Walk(false)
m.ReplaceLocal(fs)
if l1, l2 := len(m.local), len(fs); l1 != l2 {
View
@@ -1,7 +1,9 @@
package model
import (
+ "bytes"
"fmt"
+ "io/ioutil"
"log"
"os"
"path"
@@ -35,7 +37,7 @@ func tempName(name string, modified int64) string {
return path.Join(tdir, tname)
}
-func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
+func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
return nil
@@ -45,12 +47,26 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
return nil
}
- if info.Mode()&os.ModeType == 0 {
- rn, err := filepath.Rel(m.dir, p)
- if err != nil {
- return nil
+ rn, err := filepath.Rel(m.dir, p)
+ if err != nil {
+ return nil
+ }
+
+ if pn, sn := path.Split(rn); sn == ".stignore" {
+ pn := strings.Trim(pn, "/")
+ bs, _ := ioutil.ReadFile(p)
+ lines := bytes.Split(bs, []byte("\n"))
+ var patterns []string
+ for _, line := range lines {
+ if len(line) > 0 {
+ patterns = append(patterns, string(line))
+ }
}
+ ign[pn] = patterns
+ return nil
+ }
+ if info.Mode()&os.ModeType == 0 {
fi, err := os.Stat(p)
if err != nil {
return nil
@@ -94,21 +110,21 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
-func (m *Model) Walk(followSymlinks bool) []File {
- var files []File
- fn := m.genWalker(&files)
+func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]string) {
+ ignore = make(map[string][]string)
+ fn := m.genWalker(&files, ignore)
filepath.Walk(m.dir, fn)
if followSymlinks {
d, err := os.Open(m.dir)
if err != nil {
- return files
+ return
}
defer d.Close()
fis, err := d.Readdir(-1)
if err != nil {
- return files
+ return
}
for _, fi := range fis {
@@ -118,7 +134,15 @@ func (m *Model) Walk(followSymlinks bool) []File {
}
}
- return files
+ return
+}
+
+// Walk returns the list of files found in the local repository by scanning the
+// file system. Files are blockwise hashed. Patterns marked in .stignore files
+// are removed from the results.
+func (m *Model) FilteredWalk(followSymlinks bool) []File {
+ var files, ignored = m.Walk(followSymlinks)
+ return ignoreFilter(ignored, files)
}
func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
@@ -137,3 +161,21 @@ func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
func (m *Model) cleanTempFiles() {
filepath.Walk(m.dir, m.cleanTempFile)
}
+
+func ignoreFilter(patterns map[string][]string, files []File) (filtered []File) {
+nextFile:
+ for _, f := range files {
+ first, last := path.Split(f.Name)
+ for prefix, pats := range patterns {
+ if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
+ for _, pattern := range pats {
+ if match, _ := path.Match(pattern, last); match {
+ continue nextFile
+ }
+ }
+ }
+ }
+ filtered = append(filtered, f)
+ }
+ return filtered
+}
View
@@ -2,6 +2,7 @@ package model
import (
"fmt"
+ "reflect"
"testing"
"time"
)
@@ -16,9 +17,13 @@ var testdata = []struct {
{"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"},
}
+var correctIgnores = map[string][]string{
+ "": {".*", "quux"},
+}
+
func TestWalk(t *testing.T) {
m := NewModel("testdata")
- files := m.Walk(false)
+ files, ignores := m.Walk(false)
if l1, l2 := len(files), len(testdata); l1 != l2 {
t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
@@ -39,4 +44,59 @@ func TestWalk(t *testing.T) {
t.Errorf("Unrealistic modtime %d for test %d", mt, i)
}
}
+
+ if !reflect.DeepEqual(ignores, correctIgnores) {
+ t.Errorf("Incorrect ignores\n %v\n %v", correctIgnores, ignores)
+ }
+}
+
+func TestFilteredWalk(t *testing.T) {
+ m := NewModel("testdata")
+ files := m.FilteredWalk(false)
+
+ if len(files) != 2 {
+ t.Fatalf("Incorrect number of walked filtered files %d != 2", len(files))
+ }
+ if files[0].Name != "bar" {
+ t.Error("Incorrect first file", files[0])
+ }
+ if files[1].Name != "foo" {
+ t.Error("Incorrect second file", files[1])
+ }
+}
+
+func TestIgnore(t *testing.T) {
+ var patterns = map[string][]string{
+ "": {"t2"},
+ "foo": {"bar", "z*"},
+ "foo/baz": {"quux", ".*"},
+ }
+ var files = []File{
+ {Name: "foo/bar"},
+ {Name: "foo/quux"},
+ {Name: "foo/zuux"},
+ {Name: "foo/qzuux"},
+ {Name: "foo/baz/t1"},
+ {Name: "foo/baz/t2"},
+ {Name: "foo/baz/bar"},
+ {Name: "foo/baz/quuxa"},
+ {Name: "foo/baz/aquux"},
+ {Name: "foo/baz/.quux"},
+ {Name: "foo/baz/zquux"},
+ {Name: "foo/baz/quux"},
+ {Name: "foo/bazz/quux"},
+ }
+ var remaining = []File{
+ {Name: "foo/quux"},
+ {Name: "foo/qzuux"},
+ {Name: "foo/baz/t1"},
+ {Name: "foo/baz/quuxa"},
+ {Name: "foo/baz/aquux"},
+ {Name: "foo/bazz/quux"},
+ }
+
+ var filtered = ignoreFilter(patterns, files)
+ if !reflect.DeepEqual(filtered, remaining) {
+ t.Errorf("Filtering mismatch\n %v\n %v", remaining, filtered)
+ }
}

0 comments on commit 986b155

Please sign in to comment.