Skip to content

Commit

Permalink
Ignore files matching patterns in .stignore (fixes #7)
Browse files Browse the repository at this point in the history
  • Loading branch information
calmh committed Jan 6, 2014
1 parent 46d828e commit 986b155
Show file tree
Hide file tree
Showing 5 changed files with 131 additions and 20 deletions.
9 changes: 9 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,15 @@ $ syncthing --gui 127.0.0.1:8080

You then point your browser to the given address.

Excluding Files
---------------

syncthing looks for files named `.stignore` while walking the
repository. The file is expected to contain glob patterns of file names
to ignore. Patterns are matched on file name only and apply to files in
the same directory as the `.stignore` file and in directories lower down
in the hierarchy.

License
=======

Expand Down
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ func connect(myID string, addr string, nodeAddrs map[string][]string, m *model.M
}

func updateLocalModel(m *model.Model) {
files := m.Walk(!opts.NoSymlinks)
files := m.FilteredWalk(!opts.NoSymlinks)
m.ReplaceLocal(files)
saveIndex(m)
}
Expand Down
14 changes: 7 additions & 7 deletions model/model_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func init() {

func TestUpdateLocal(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

if len(m.need) > 0 {
Expand Down Expand Up @@ -100,7 +100,7 @@ func TestUpdateLocal(t *testing.T) {

func TestRemoteUpdateExisting(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

newFile := protocol.FileInfo{
Expand All @@ -117,7 +117,7 @@ func TestRemoteUpdateExisting(t *testing.T) {

func TestRemoteAddNew(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

newFile := protocol.FileInfo{
Expand All @@ -134,7 +134,7 @@ func TestRemoteAddNew(t *testing.T) {

func TestRemoteUpdateOld(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

oldTimeStamp := int64(1234)
Expand All @@ -152,7 +152,7 @@ func TestRemoteUpdateOld(t *testing.T) {

func TestRemoteIndexUpdate(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

foo := protocol.FileInfo{
Expand Down Expand Up @@ -185,7 +185,7 @@ func TestRemoteIndexUpdate(t *testing.T) {

func TestDelete(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

if l1, l2 := len(m.local), len(fs); l1 != l2 {
Expand Down Expand Up @@ -275,7 +275,7 @@ func TestDelete(t *testing.T) {

func TestForgetNode(t *testing.T) {
m := NewModel("testdata")
fs := m.Walk(false)
fs, _ := m.Walk(false)
m.ReplaceLocal(fs)

if l1, l2 := len(m.local), len(fs); l1 != l2 {
Expand Down
64 changes: 53 additions & 11 deletions model/walk.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package model

import (
"bytes"
"fmt"
"io/ioutil"
"log"
"os"
"path"
Expand Down Expand Up @@ -35,7 +37,7 @@ func tempName(name string, modified int64) string {
return path.Join(tdir, tname)
}

func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
func (m *Model) genWalker(res *[]File, ign map[string][]string) filepath.WalkFunc {
return func(p string, info os.FileInfo, err error) error {
if err != nil {
return nil
Expand All @@ -45,12 +47,26 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {
return nil
}

if info.Mode()&os.ModeType == 0 {
rn, err := filepath.Rel(m.dir, p)
if err != nil {
return nil
rn, err := filepath.Rel(m.dir, p)
if err != nil {
return nil
}

if pn, sn := path.Split(rn); sn == ".stignore" {
pn := strings.Trim(pn, "/")
bs, _ := ioutil.ReadFile(p)
lines := bytes.Split(bs, []byte("\n"))
var patterns []string
for _, line := range lines {
if len(line) > 0 {
patterns = append(patterns, string(line))
}
}
ign[pn] = patterns
return nil
}

if info.Mode()&os.ModeType == 0 {
fi, err := os.Stat(p)
if err != nil {
return nil
Expand Down Expand Up @@ -94,21 +110,21 @@ func (m *Model) genWalker(res *[]File) filepath.WalkFunc {

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed.
func (m *Model) Walk(followSymlinks bool) []File {
var files []File
fn := m.genWalker(&files)
func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]string) {
ignore = make(map[string][]string)
fn := m.genWalker(&files, ignore)
filepath.Walk(m.dir, fn)

if followSymlinks {
d, err := os.Open(m.dir)
if err != nil {
return files
return
}
defer d.Close()

fis, err := d.Readdir(-1)
if err != nil {
return files
return
}

for _, fi := range fis {
Expand All @@ -118,7 +134,15 @@ func (m *Model) Walk(followSymlinks bool) []File {
}
}

return files
return
}

// Walk returns the list of files found in the local repository by scanning the
// file system. Files are blockwise hashed. Patterns marked in .stignore files
// are removed from the results.
func (m *Model) FilteredWalk(followSymlinks bool) []File {
var files, ignored = m.Walk(followSymlinks)
return ignoreFilter(ignored, files)
}

func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
Expand All @@ -137,3 +161,21 @@ func (m *Model) cleanTempFile(path string, info os.FileInfo, err error) error {
func (m *Model) cleanTempFiles() {
filepath.Walk(m.dir, m.cleanTempFile)
}

func ignoreFilter(patterns map[string][]string, files []File) (filtered []File) {
nextFile:
for _, f := range files {
first, last := path.Split(f.Name)
for prefix, pats := range patterns {
if len(prefix) == 0 || prefix == first || strings.HasPrefix(first, prefix+"/") {
for _, pattern := range pats {
if match, _ := path.Match(pattern, last); match {
continue nextFile
}
}
}
}
filtered = append(filtered, f)
}
return filtered
}
62 changes: 61 additions & 1 deletion model/walk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package model

import (
"fmt"
"reflect"
"testing"
"time"
)
Expand All @@ -16,9 +17,13 @@ var testdata = []struct {
{"foo", 7, "aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"},
}

var correctIgnores = map[string][]string{
"": {".*", "quux"},
}

func TestWalk(t *testing.T) {
m := NewModel("testdata")
files := m.Walk(false)
files, ignores := m.Walk(false)

if l1, l2 := len(files), len(testdata); l1 != l2 {
t.Fatalf("Incorrect number of walked files %d != %d", l1, l2)
Expand All @@ -39,4 +44,59 @@ func TestWalk(t *testing.T) {
t.Errorf("Unrealistic modtime %d for test %d", mt, i)
}
}

if !reflect.DeepEqual(ignores, correctIgnores) {
t.Errorf("Incorrect ignores\n %v\n %v", correctIgnores, ignores)
}
}

func TestFilteredWalk(t *testing.T) {
m := NewModel("testdata")
files := m.FilteredWalk(false)

if len(files) != 2 {
t.Fatalf("Incorrect number of walked filtered files %d != 2", len(files))
}
if files[0].Name != "bar" {
t.Error("Incorrect first file", files[0])
}
if files[1].Name != "foo" {
t.Error("Incorrect second file", files[1])
}
}

func TestIgnore(t *testing.T) {
var patterns = map[string][]string{
"": {"t2"},
"foo": {"bar", "z*"},
"foo/baz": {"quux", ".*"},
}
var files = []File{
{Name: "foo/bar"},
{Name: "foo/quux"},
{Name: "foo/zuux"},
{Name: "foo/qzuux"},
{Name: "foo/baz/t1"},
{Name: "foo/baz/t2"},
{Name: "foo/baz/bar"},
{Name: "foo/baz/quuxa"},
{Name: "foo/baz/aquux"},
{Name: "foo/baz/.quux"},
{Name: "foo/baz/zquux"},
{Name: "foo/baz/quux"},
{Name: "foo/bazz/quux"},
}
var remaining = []File{
{Name: "foo/quux"},
{Name: "foo/qzuux"},
{Name: "foo/baz/t1"},
{Name: "foo/baz/quuxa"},
{Name: "foo/baz/aquux"},
{Name: "foo/bazz/quux"},
}

var filtered = ignoreFilter(patterns, files)
if !reflect.DeepEqual(filtered, remaining) {
t.Errorf("Filtering mismatch\n %v\n %v", remaining, filtered)
}
}

0 comments on commit 986b155

Please sign in to comment.