Skip to content
Permalink
Browse files

Ignore files matching patterns in .stignore (fixes syncthing#7)

  • Loading branch information
calmh committed Jan 6, 2014
1 parent 46d828e commit 986b15573a66f95beae53b10371ba50ab147124a
Showing with 131 additions and 20 deletions.
  1. +9 −0 README.md
  2. +1 −1 main.go
  3. +7 −7 model/model_test.go
  4. +53 −11 model/walk.go
  5. +61 −1 model/walk_test.go
@@ -186,6 +186,15 @@ $ syncthing --gui 127.0.0.1:8080

You then point your browser to the given address.

Excluding Files
---------------

syncthing looks for files named `.stignore` while walking the
repository. The file is expected to contain glob patterns of file names
to ignore. Patterns are matched on file name only and apply to files in
the same directory as the `.stignore` file and in directories lower down
in the hierarchy.

License
=======

@@ -333,7 +333,7 @@ func connect(myID string, addr string, nodeAddrs map[string][]string, m *model.M
}

func updateLocalModel(m *model.Model) {
files := m.Walk(!opts.NoSymlinks)
files := m.FilteredWalk(!opts.NoSymlinks)
m.ReplaceLocal(files)
saveIndex(m)
}
@@ -58,7 +58,7 @@ func init() {

func TestUpdateLocal(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

if len(m.need) > 0 {
@@ -100,7 +100,7 @@ func TestUpdateLocal(t *testing.T) {

func TestRemoteUpdateExisting(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

newFile := protocol.FileInfo{
@@ -117,7 +117,7 @@ func TestRemoteUpdateExisting(t *testing.T) {

func TestRemoteAddNew(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

newFile := protocol.FileInfo{
@@ -134,7 +134,7 @@ func TestRemoteAddNew(t *testing.T) {

func TestRemoteUpdateOld(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

oldTimeStamp := int64(1234)
@@ -152,7 +152,7 @@ func TestRemoteUpdateOld(t *testing.T) {

func TestRemoteIndexUpdate(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

foo := protocol.FileInfo{
@@ -185,7 +185,7 @@ func TestRemoteIndexUpdate(t *testing.T) {

func TestDelete(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -275,7 +275,7 @@ func TestDelete(t *testing.T) {

func TestForgetNode(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

if l1, l2 := len(m.local), len(fs); l1 != l2 {
@@ -1,7 +1,9 @@
package model

import (
"bytes"
"fmt"
"io/ioutil"
"log"
"os"
"path"
@@ -35,7 +37,7 @@ func tempName(name string, modified int64) string {
return path.Join(tdir, tname)
}

func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
return nil
@@ -45,12 +47,26 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
return nil
}

if info.Mode()&os.ModeType == 0 {
rn, err := filepath.Rel(m.dir, p)
if err != nil {
return nil
rn, err := filepath.Rel(m.dir, p)
if err != nil {
return nil
}

if pn, sn := path.Split(rn); sn == ".stignore" {
pn := strings.Trim(pn, "/")
bs, _ := ioutil.ReadFile(p)
lines := bytes.Split(bs, []byte("\n"))
var patterns []string
for _, line := range lines {
if len(line) > 0 {
patterns = append(patterns, string(line))
}
}
ign[pn] = patterns
return nil
}

if info.Mode()&os.ModeType == 0 {
fi, err := os.Stat(p)
if err != nil {
return nil
@@ -94,21 +110,21 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (m *Model) Walk(followSymlinks bool) []File {
var files []File
fn := m.genWalker(&files)
func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]string) {
ignore = make(map[string][]string)
fn := m.genWalker(&files, ignore)
filepath.Walk(m.dir, fn)

if followSymlinks {
d, err := os.Open(m.dir)
if err != nil {
return files
return
}
defer d.Close()

fis, err := d.Readdir(-1)
if err != nil {
return files
return
}

for _, fi := range fis {
@@ -118,7 +134,15 @@ func (m *Model) Walk(followSymlinks bool) []File {
}
}

return files
return
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed. Patterns marked in .stignore files
// are removed from the results.
func (m *Model) FilteredWalk(followSymlinks bool) []File {
var files, ignored = m.Walk(followSymlinks)
return ignoreFilter(ignored, files)
}

func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
@@ -137,3 +161,21 @@ func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
func (m *Model) cleanTempFiles() {
filepath.Walk(m.dir, m.cleanTempFile)
}

func ignoreFilter(patterns map[string][]string, files []File) (filtered []File) {
nextFile:
for _, f := range files {
first, last := path.Split(f.Name)
for prefix, pats := range patterns {
if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
for _, pattern := range pats {
if match, _ := path.Match(pattern, last); match {
continue nextFile
}
}
}
}
filtered = append(filtered, f)
}
return filtered
}
@@ -2,6 +2,7 @@ package model

import (
"fmt"
"reflect"
"testing"
"time"
)
@@ -16,9 +17,13 @@ var testdata = []struct {
{"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"},
}

var correctIgnores = map[string][]string{
"": {".*", "quux"},
}

func TestWalk(t *testing.T) {
m := NewModel("testdata")
files := m.Walk(false)
files, ignores := m.Walk(false)

if l1, l2 := len(files), len(testdata); l1 != l2 {
t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
@@ -39,4 +44,59 @@ func TestWalk(t *testing.T) {
t.Errorf("Unrealistic modtime %d for test %d", mt, i)
}
}

if !reflect.DeepEqual(ignores, correctIgnores) {
t.Errorf("Incorrect ignores\n %v\n %v", correctIgnores, ignores)
}
}

func TestFilteredWalk(t *testing.T) {
m := NewModel("testdata")
files := m.FilteredWalk(false)

if len(files) != 2 {
t.Fatalf("Incorrect number of walked filtered files %d != 2", len(files))
}
if files[0].Name != "bar" {
t.Error("Incorrect first file", files[0])
}
if files[1].Name != "foo" {
t.Error("Incorrect second file", files[1])
}
}

func TestIgnore(t *testing.T) {
var patterns = map[string][]string{
"": {"t2"},
"foo": {"bar", "z*"},
"foo/baz": {"quux", ".*"},
}
var files = []File{
{Name: "foo/bar"},
{Name: "foo/quux"},
{Name: "foo/zuux"},
{Name: "foo/qzuux"},
{Name: "foo/baz/t1"},
{Name: "foo/baz/t2"},
{Name: "foo/baz/bar"},
{Name: "foo/baz/quuxa"},
{Name: "foo/baz/aquux"},
{Name: "foo/baz/.quux"},
{Name: "foo/baz/zquux"},
{Name: "foo/baz/quux"},
{Name: "foo/bazz/quux"},
}
var remaining = []File{
{Name: "foo/quux"},
{Name: "foo/qzuux"},
{Name: "foo/baz/t1"},
{Name: "foo/baz/quuxa"},
{Name: "foo/baz/aquux"},
{Name: "foo/bazz/quux"},
}

var filtered = ignoreFilter(patterns, files)
if !reflect.DeepEqual(filtered, remaining) {
t.Errorf("Filtering mismatch\n %v\n %v", remaining, filtered)
}
}

0 comments on commit 986b155

Please sign in to comment.
You can’t perform that action at this time.