From d23b8e0d97b1449c67e4cd5e41c610b876ea67fd Mon Sep 17 00:00:00 2001 From: Unrud Date: Sat, 17 Dec 2016 08:52:33 +0100 Subject: [PATCH 01/13] Check file names for conflicts on Windows --- lib/model/model.go | 6 ++++ lib/model/rwfolder.go | 14 +++++++++ lib/osutil/name_conflict.go | 16 ++++++++++ lib/osutil/name_conflict_windows.go | 48 +++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 lib/osutil/name_conflict.go create mode 100644 lib/osutil/name_conflict_windows.go diff --git a/lib/model/model.go b/lib/model/model.go index b06743702e9..b42133f8e5c 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -121,6 +121,7 @@ var ( errDevicePaused = errors.New("device is paused") errDeviceIgnored = errors.New("device is ignored") errNotRelative = errors.New("not a relative path") + errNameConflict = errors.New("filename collides with existing file") ) // NewModel creates and starts a new model. The model starts in read-only mode, @@ -1163,6 +1164,11 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset return protocol.ErrNoSuchFile } + if !osutil.CheckNameConflict(folderPath, name) { + l.Debugf("%v REQ(in) for file not in dir: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf)) + return protocol.ErrNoSuchFile + } + // Only check temp files if the flag is set, and if we are set to advertise // the temp indexes. if fromTemporary && !folderCfg.DisableTempIndexes { diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index ca4b85d0ced..081b7e5860b 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -436,6 +436,13 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int { continue } + // Verify that we handle the right thing and not something whose name + // collides. + if !osutil.CheckNameConflict(f.dir, fi.Name) { + f.newError(fi.Name, errNameConflict) + continue + } + switch { case fi.IsDeleted(): // A deleted file, directory or symlink @@ -525,6 +532,13 @@ nextFile: continue } + // Verify that we handle the right thing and not something whose name + // collides. + if !osutil.CheckNameConflict(f.dir, fi.Name) { + f.newError(fi.Name, errNameConflict) + continue + } + // Check our list of files to be removed for a match, in which case // we can just do a rename instead. key := string(fi.Blocks[0].Hash) diff --git a/lib/osutil/name_conflict.go b/lib/osutil/name_conflict.go new file mode 100644 index 00000000000..37248990ea9 --- /dev/null +++ b/lib/osutil/name_conflict.go @@ -0,0 +1,16 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows + +package osutil + +// CheckNameConflict returns true if every path component of name up to and +// including filepath.Join(base, name) doesn't conflict with any existing +// files or folders with different names. +func CheckNameConflict(base, name string) bool { + return true +} diff --git a/lib/osutil/name_conflict_windows.go b/lib/osutil/name_conflict_windows.go new file mode 100644 index 00000000000..f242e2d87c6 --- /dev/null +++ b/lib/osutil/name_conflict_windows.go @@ -0,0 +1,48 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows + +package osutil + +import ( + "os" + "path/filepath" + "strings" + "syscall" +) + +// CheckNameConflict returns true if every path component of name up to and +// including filepath.Join(base, name) doesn't conflict with any existing +// files or folders with different names. Base and name must both be clean and +// name must be relative to base. +func CheckNameConflict(base, name string) bool { + // Conflicts can be caused by different casing (e.g. foo and FOO) or + // by the use of short names (e.g. foo.barbaz and FOO~1.BAR). + path := base + parts := strings.Split(name, string(os.PathSeparator)) + for _, part := range parts { + path = filepath.Join(path, part) + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return false + } + var data syscall.Win32finddata + handle, err := syscall.FindFirstFile(pathp, &data) + if err == syscall.ERROR_FILE_NOT_FOUND { + return true + } + if err != nil { + return false + } + syscall.Close(handle) + fileName := syscall.UTF16ToString(data.FileName[:]) + if part != fileName { + return false + } + } + return true +} From 6fa90f7e7886fe6cb6d2e4fe2bdc8abe027c7507 Mon Sep 17 00:00:00 2001 From: Unrud Date: Sat, 17 Dec 2016 08:53:28 +0100 Subject: [PATCH 02/13] Verify file before opening --- lib/model/rwfolder.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index 081b7e5860b..f7c354d13ef 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -1297,6 +1297,16 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch if err != nil { return false } + // The following checks are racy + if err := osutil.TraversesSymlink(folderRoots[folder], filepath.Dir(file)); err != nil { + return false + } + if !osutil.CheckNameConflict(folderRoots[folder], file) { + return false + } + if info, err := osutil.Lstat(inFile); err != nil || !info.Mode().IsRegular() { + return false + } fd, err := os.Open(inFile) if err != nil { return false From 11e87bab2ec56b0350df3fcf2c9b2e06465e76d0 Mon Sep 17 00:00:00 2001 From: Unrud Date: Sat, 17 Dec 2016 09:29:56 +0100 Subject: [PATCH 03/13] Close handle correctly --- lib/osutil/name_conflict_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/osutil/name_conflict_windows.go b/lib/osutil/name_conflict_windows.go index f242e2d87c6..1014c92eb9c 100644 --- a/lib/osutil/name_conflict_windows.go +++ b/lib/osutil/name_conflict_windows.go @@ -38,7 +38,7 @@ func CheckNameConflict(base, name string) bool { if err != nil { return false } - syscall.Close(handle) + syscall.FindClose(handle) fileName := syscall.UTF16ToString(data.FileName[:]) if part != fileName { return false From 0ad273cdcc9d5abc1350e8f1d266af04730036d8 Mon Sep 17 00:00:00 2001 From: Unrud Date: Sat, 17 Dec 2016 09:31:08 +0100 Subject: [PATCH 04/13] Use same documentation --- lib/osutil/name_conflict.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/osutil/name_conflict.go b/lib/osutil/name_conflict.go index 37248990ea9..a3edd8f0637 100644 --- a/lib/osutil/name_conflict.go +++ b/lib/osutil/name_conflict.go @@ -10,7 +10,8 @@ package osutil // CheckNameConflict returns true if every path component of name up to and // including filepath.Join(base, name) doesn't conflict with any existing -// files or folders with different names. +// files or folders with different names. Base and name must both be clean and +// name must be relative to base. func CheckNameConflict(base, name string) bool { return true } From d7dfe045f1c13bde7643f4ad64f758a622394f4a Mon Sep 17 00:00:00 2001 From: Unrud Date: Sat, 17 Dec 2016 18:13:22 +0100 Subject: [PATCH 05/13] Verify paths from external tools like inotify Protect against scanning of short names on Windows. (maybe caused #3800) Protect against the possibility of traversing Symlinks. --- lib/model/model.go | 23 ++++++++++++++------- lib/model/model_test.go | 45 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/lib/model/model.go b/lib/model/model.go index b42133f8e5c..df8dbf7b839 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -1792,9 +1792,16 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error // Clean the list of subitems to ensure that we start at a known // directory, and don't scan subdirectories of things we've already // scanned. - subDirs = unifySubs(subDirs, func(f string) bool { - _, ok := fs.Get(protocol.LocalDeviceID, f) - return ok + subDirs = unifySubs(subDirs, func(dir string) bool { + if _, ok := fs.Get(protocol.LocalDeviceID, dir); !ok { + return false + } + if !osutil.IsDir(folderCfg.Path(), dir) { + return false + } + return true + }, func(name string) bool { + return osutil.CheckNameConflict(folderCfg.Path(), name) }) // The cancel channel is closed whenever we return (such as from an error), @@ -2567,19 +2574,21 @@ func readOffsetIntoBuf(file string, offset int64, buf []byte) error { // The exists function is expected to return true for all known paths // (excluding "" and ".") -func unifySubs(dirs []string, exists func(dir string) bool) []string { - subs := trimUntilParentKnown(dirs, exists) +func unifySubs(dirs []string, exists func(dir string) bool, + isSafe func(name string) bool) []string { + subs := trimUntilParentKnownAndSafe(dirs, exists, isSafe) sort.Strings(subs) return simplifySortedPaths(subs) } -func trimUntilParentKnown(dirs []string, exists func(dir string) bool) []string { +func trimUntilParentKnownAndSafe(dirs []string, exists func(dir string) bool, + isSafe func(name string) bool) []string { var subs []string for _, sub := range dirs { for sub != "" && !ignore.IsInternal(sub) { sub = filepath.Clean(sub) parent := filepath.Dir(sub) - if parent == "." || exists(parent) { + if (parent == "." || exists(parent)) && isSafe(sub) { break } sub = parent diff --git a/lib/model/model_test.go b/lib/model/model_test.go index a95884e9817..be453419fd8 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -1641,18 +1641,21 @@ func TestUnifySubs(t *testing.T) { cases := []struct { in []string // input to unifySubs exists []string // paths that exist in the database + unsafe []string // paths that are not safe (e.g. name conflict) out []string // expected output }{ { // 0. trailing slashes are cleaned, known paths are just passed on []string{"foo/", "bar//"}, []string{"foo", "bar"}, + nil, []string{"bar", "foo"}, // the output is sorted }, { // 1. "foo/bar" gets trimmed as it's covered by foo []string{"foo", "bar/", "foo/bar/"}, []string{"foo", "bar"}, + nil, []string{"bar", "foo"}, }, { @@ -1660,12 +1663,14 @@ func TestUnifySubs(t *testing.T) { []string{"foo", ""}, []string{"foo"}, nil, + nil, }, { // 3. "foo/bar" is unknown, but it's kept // because its parent is known []string{"foo/bar"}, []string{"foo"}, + nil, []string{"foo/bar"}, }, { @@ -1673,12 +1678,14 @@ func TestUnifySubs(t *testing.T) { // "usr/lib" is not a prefix of "usr/libexec" []string{"usr/lib", "usr/libexec"}, []string{"usr", "usr/lib", "usr/libexec"}, + nil, []string{"usr/lib", "usr/libexec"}, }, { // 5. "usr/lib" is a prefix of "usr/lib/exec" []string{"usr/lib", "usr/lib/exec"}, []string{"usr", "usr/lib", "usr/libexec"}, + nil, []string{"usr/lib"}, }, { @@ -1686,6 +1693,7 @@ func TestUnifySubs(t *testing.T) { // verbatim even though they are unknown []string{".stfolder", ".stignore"}, []string{}, + nil, []string{".stfolder", ".stignore"}, }, { @@ -1693,6 +1701,7 @@ func TestUnifySubs(t *testing.T) { // scan []string{".stfolder", ".stignore", "foo/bar"}, []string{}, + nil, []string{".stfolder", ".stignore", "foo"}, }, { @@ -1700,12 +1709,35 @@ func TestUnifySubs(t *testing.T) { nil, []string{"foo"}, nil, + nil, }, { // 9. empty list of subs []string{}, []string{"foo"}, nil, + nil, + }, + { + // 10. name conflict + []string{"FOO"}, + nil, + []string{"FOO"}, + nil, + }, + { + // 11. name conflict in first component + []string{"FOO/bar"}, + []string{"FOO", "FOO/bar"}, + []string{"FOO", "FOO/bar"}, + nil, + }, + { + // 12. name conflict in last component + []string{"foo/BAR"}, + []string{"foo", "foo/BAR"}, + []string{"foo/BAR"}, + []string{"foo"}, }, } @@ -1718,6 +1750,9 @@ func TestUnifySubs(t *testing.T) { for j, p := range cases[i].exists { cases[i].exists[j] = filepath.FromSlash(p) } + for j, p := range cases[i].unsafe { + cases[i].unsafe[j] = filepath.FromSlash(p) + } for j, p := range cases[i].out { cases[i].out[j] = filepath.FromSlash(p) } @@ -1733,8 +1768,16 @@ func TestUnifySubs(t *testing.T) { } return false } + isSafe := func(f string) bool { + for _, e := range tc.unsafe { + if f == e { + return false + } + } + return true + } - out := unifySubs(tc.in, exists) + out := unifySubs(tc.in, exists, isSafe) if diff, equal := messagediff.PrettyDiff(tc.out, out); !equal { t.Errorf("Case %d failed; got %v, expected %v, diff:\n%s", i, out, tc.out, diff) } From 8444b3572cd14f79d5698eec1c2ce45a94729630 Mon Sep 17 00:00:00 2001 From: Unrud Date: Wed, 21 Dec 2016 06:15:58 +0100 Subject: [PATCH 06/13] Revert "Verify paths from external tools like inotify" This reverts commit bf3da33b96b440e29a3028c004e6bd7305ed351f. --- lib/model/model.go | 23 +++++++-------------- lib/model/model_test.go | 45 +---------------------------------------- 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/lib/model/model.go b/lib/model/model.go index df8dbf7b839..b42133f8e5c 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -1792,16 +1792,9 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error // Clean the list of subitems to ensure that we start at a known // directory, and don't scan subdirectories of things we've already // scanned. - subDirs = unifySubs(subDirs, func(dir string) bool { - if _, ok := fs.Get(protocol.LocalDeviceID, dir); !ok { - return false - } - if !osutil.IsDir(folderCfg.Path(), dir) { - return false - } - return true - }, func(name string) bool { - return osutil.CheckNameConflict(folderCfg.Path(), name) + subDirs = unifySubs(subDirs, func(f string) bool { + _, ok := fs.Get(protocol.LocalDeviceID, f) + return ok }) // The cancel channel is closed whenever we return (such as from an error), @@ -2574,21 +2567,19 @@ func readOffsetIntoBuf(file string, offset int64, buf []byte) error { // The exists function is expected to return true for all known paths // (excluding "" and ".") -func unifySubs(dirs []string, exists func(dir string) bool, - isSafe func(name string) bool) []string { - subs := trimUntilParentKnownAndSafe(dirs, exists, isSafe) +func unifySubs(dirs []string, exists func(dir string) bool) []string { + subs := trimUntilParentKnown(dirs, exists) sort.Strings(subs) return simplifySortedPaths(subs) } -func trimUntilParentKnownAndSafe(dirs []string, exists func(dir string) bool, - isSafe func(name string) bool) []string { +func trimUntilParentKnown(dirs []string, exists func(dir string) bool) []string { var subs []string for _, sub := range dirs { for sub != "" && !ignore.IsInternal(sub) { sub = filepath.Clean(sub) parent := filepath.Dir(sub) - if (parent == "." || exists(parent)) && isSafe(sub) { + if parent == "." || exists(parent) { break } sub = parent diff --git a/lib/model/model_test.go b/lib/model/model_test.go index be453419fd8..a95884e9817 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -1641,21 +1641,18 @@ func TestUnifySubs(t *testing.T) { cases := []struct { in []string // input to unifySubs exists []string // paths that exist in the database - unsafe []string // paths that are not safe (e.g. name conflict) out []string // expected output }{ { // 0. trailing slashes are cleaned, known paths are just passed on []string{"foo/", "bar//"}, []string{"foo", "bar"}, - nil, []string{"bar", "foo"}, // the output is sorted }, { // 1. "foo/bar" gets trimmed as it's covered by foo []string{"foo", "bar/", "foo/bar/"}, []string{"foo", "bar"}, - nil, []string{"bar", "foo"}, }, { @@ -1663,14 +1660,12 @@ func TestUnifySubs(t *testing.T) { []string{"foo", ""}, []string{"foo"}, nil, - nil, }, { // 3. "foo/bar" is unknown, but it's kept // because its parent is known []string{"foo/bar"}, []string{"foo"}, - nil, []string{"foo/bar"}, }, { @@ -1678,14 +1673,12 @@ func TestUnifySubs(t *testing.T) { // "usr/lib" is not a prefix of "usr/libexec" []string{"usr/lib", "usr/libexec"}, []string{"usr", "usr/lib", "usr/libexec"}, - nil, []string{"usr/lib", "usr/libexec"}, }, { // 5. "usr/lib" is a prefix of "usr/lib/exec" []string{"usr/lib", "usr/lib/exec"}, []string{"usr", "usr/lib", "usr/libexec"}, - nil, []string{"usr/lib"}, }, { @@ -1693,7 +1686,6 @@ func TestUnifySubs(t *testing.T) { // verbatim even though they are unknown []string{".stfolder", ".stignore"}, []string{}, - nil, []string{".stfolder", ".stignore"}, }, { @@ -1701,7 +1693,6 @@ func TestUnifySubs(t *testing.T) { // scan []string{".stfolder", ".stignore", "foo/bar"}, []string{}, - nil, []string{".stfolder", ".stignore", "foo"}, }, { @@ -1709,35 +1700,12 @@ func TestUnifySubs(t *testing.T) { nil, []string{"foo"}, nil, - nil, }, { // 9. empty list of subs []string{}, []string{"foo"}, nil, - nil, - }, - { - // 10. name conflict - []string{"FOO"}, - nil, - []string{"FOO"}, - nil, - }, - { - // 11. name conflict in first component - []string{"FOO/bar"}, - []string{"FOO", "FOO/bar"}, - []string{"FOO", "FOO/bar"}, - nil, - }, - { - // 12. name conflict in last component - []string{"foo/BAR"}, - []string{"foo", "foo/BAR"}, - []string{"foo/BAR"}, - []string{"foo"}, }, } @@ -1750,9 +1718,6 @@ func TestUnifySubs(t *testing.T) { for j, p := range cases[i].exists { cases[i].exists[j] = filepath.FromSlash(p) } - for j, p := range cases[i].unsafe { - cases[i].unsafe[j] = filepath.FromSlash(p) - } for j, p := range cases[i].out { cases[i].out[j] = filepath.FromSlash(p) } @@ -1768,16 +1733,8 @@ func TestUnifySubs(t *testing.T) { } return false } - isSafe := func(f string) bool { - for _, e := range tc.unsafe { - if f == e { - return false - } - } - return true - } - out := unifySubs(tc.in, exists, isSafe) + out := unifySubs(tc.in, exists) if diff, equal := messagediff.PrettyDiff(tc.out, out); !equal { t.Errorf("Case %d failed; got %v, expected %v, diff:\n%s", i, out, tc.out, diff) } From c56b36140db0e65ab3a3bb3a6a9d38b1909cdd47 Mon Sep 17 00:00:00 2001 From: Unrud Date: Wed, 21 Dec 2016 06:19:13 +0100 Subject: [PATCH 07/13] Check sub paths in scanner Prevent scanner from following Symlinks and scanning the contents of colliding paths on Windows (e.g. scanning "foo" for the sub path "Foo") --- lib/scanner/walk.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index 5c1e2649cf4..83e7d987d49 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -120,6 +120,14 @@ func (w *walker) walk() (chan protocol.FileInfo, error) { filepath.Walk(w.Dir, hashFiles) } else { for _, sub := range w.Subs { + if err := osutil.TraversesSymlink(w.Dir, filepath.Dir(sub)); err != nil { + l.Infoln("Skipping sub path that traverses symlink", w.Dir, sub) + continue + } + if !osutil.CheckNameConflict(w.Dir, sub) { + l.Infoln("Skipping sub path that collides", w.Dir, sub) + continue + } filepath.Walk(filepath.Join(w.Dir, sub), hashFiles) } } From 10f86af23fc854f4f9db439eadb1c8716a459b89 Mon Sep 17 00:00:00 2001 From: Unrud Date: Wed, 21 Dec 2016 09:44:47 +0100 Subject: [PATCH 08/13] Mark conflicting paths as deleted --- lib/model/model.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/model/model.go b/lib/model/model.go index b42133f8e5c..c487f6a454c 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -1905,7 +1905,12 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error // The file is valid and not deleted. Lets check if it's // still here. - if _, err := mtimefs.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil { + var exists bool + if err := osutil.TraversesSymlink(folderCfg.Path(), filepath.Dir(f.Name)); err != nil { + exists = false + } else if !osutil.CheckNameConflict(folderCfg.Path(), f.Name) { + exists = false + } else if _, err := mtimefs.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil { // We don't specifically verify that the error is // os.IsNotExist because there is a corner case when a // directory is suddenly transformed into a file. When that @@ -1913,6 +1918,11 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error // file) are deleted but will return a confusing error ("not a // directory") when we try to Lstat() them. + exists = false + } else { + exists = true + } + if !exists { nf := protocol.FileInfo{ Name: f.Name, Type: f.Type, From e25fb9eebf52ce0a2d2b471f7a1e196a228f5d49 Mon Sep 17 00:00:00 2001 From: Unrud Date: Wed, 21 Dec 2016 21:49:16 +0100 Subject: [PATCH 09/13] Add tests for osutil.CheckNameConflict --- lib/osutil/name_conflict_windows_test.go | 108 +++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 lib/osutil/name_conflict_windows_test.go diff --git a/lib/osutil/name_conflict_windows_test.go b/lib/osutil/name_conflict_windows_test.go new file mode 100644 index 00000000000..f8e6aacef7f --- /dev/null +++ b/lib/osutil/name_conflict_windows_test.go @@ -0,0 +1,108 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows + +package osutil_test + +import ( + "os" + "path" + "path/filepath" + "syscall" + "testing" + "unsafe" + + "github.com/syncthing/syncthing/lib/osutil" +) + +func TestCheckNameConflictCasing(t *testing.T) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.MkdirAll("testdata/Foo/BAR/baz", 0755) + // check if the file system is case-sensitive + if _, err := os.Lstat("testdata/foo"); err != nil { + t.Skip("pointless test") + return + } + + cases := []struct { + name string + conflictFree bool + }{ + // Exists + {"Foo", true}, + {"Foo/BAR", true}, + {"Foo/BAR/baz", true}, + // Doesn't exist + {"bar", true}, + {"Foo/baz", true}, + // Conflicts + {"foo", false}, + {"foo/BAR", false}, + {"Foo/bar", false}, + {"Foo/BAR/BAZ", false}, + } + + for _, tc := range cases { + nativeName := filepath.FromSlash(tc.name) + if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { + t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) + } + } +} + +func TestCheckNameConflictShortName(t *testing.T) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.MkdirAll("testdata/foobarbaz/qux", 0755) + ppath, err := syscall.UTF16PtrFromString("testdata/foobarbaz") + if err != nil { + t.Fatal("unexpected error", err) + } + // check if the file system supports short names + bufferSize, err := syscall.GetShortPathName(ppath, nil, 0) + if err != nil { + t.Skip("pointless test") + return + } + + // get the short name + buffer := make([]uint16, bufferSize) + length, err := syscall.GetShortPathName(ppath, + (*uint16)(unsafe.Pointer(&buffer[0])), bufferSize) + if err != nil { + t.Fatal("unexpected error", err) + } + // on success length doesn't contain the terminating null character + if bufferSize != length+1 { + t.Fatal("length of short name changed") + } + shortName := filepath.Base(syscall.UTF16ToString(buffer)) + + cases := []struct { + name string + conflictFree bool + }{ + // Exists + {"foobarbaz", true}, + {"foobarbaz/qux", true}, + // Doesn't exist + {"foo", true}, + {"foobarbaz/quux", true}, + // Conflicts + {shortName, false}, + {path.Join(shortName, "qux"), false}, + {path.Join(shortName, "quux"), false}, + } + + for _, tc := range cases { + nativeName := filepath.FromSlash(tc.name) + if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { + t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) + } + } +} From 6d0b7d50fc760e6c43c43ce34c6b3053b4c01de8 Mon Sep 17 00:00:00 2001 From: Unrud Date: Tue, 27 Dec 2016 15:52:03 +0100 Subject: [PATCH 10/13] Implement CheckNameConflict for Unix Make CheckNameConflict system-independent and move system-dependent code into function FindRealFileName. --- lib/osutil/name_conflict.go | 26 ++++++++- lib/osutil/name_conflict_test.go | 70 ++++++++++++++++++++++++ lib/osutil/name_conflict_unix.go | 57 +++++++++++++++++++ lib/osutil/name_conflict_windows.go | 47 ++++++---------- lib/osutil/name_conflict_windows_test.go | 37 ------------- 5 files changed, 169 insertions(+), 68 deletions(-) create mode 100644 lib/osutil/name_conflict_test.go create mode 100644 lib/osutil/name_conflict_unix.go diff --git a/lib/osutil/name_conflict.go b/lib/osutil/name_conflict.go index a3edd8f0637..8341cf78eef 100644 --- a/lib/osutil/name_conflict.go +++ b/lib/osutil/name_conflict.go @@ -4,14 +4,36 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. -// +build !windows - package osutil +import ( + "os" + "path/filepath" + "strings" +) + // CheckNameConflict returns true if every path component of name up to and // including filepath.Join(base, name) doesn't conflict with any existing // files or folders with different names. Base and name must both be clean and // name must be relative to base. func CheckNameConflict(base, name string) bool { + // Conflicts depend on the OS and file system. + subname := "." + parts := strings.Split(name, string(os.PathSeparator)) + for _, part := range parts { + subname = filepath.Join(subname, part) + fileName, err := FindRealFileName(base, subname) + if err != nil { + return false + } + if fileName == "" { + // doesn't exist + return true + } + if fileName != part { + // conflict + return false + } + } return true } diff --git a/lib/osutil/name_conflict_test.go b/lib/osutil/name_conflict_test.go new file mode 100644 index 00000000000..1ae70659dfa --- /dev/null +++ b/lib/osutil/name_conflict_test.go @@ -0,0 +1,70 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package osutil_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/syncthing/syncthing/lib/osutil" +) + +func TestCheckNameConflict(t *testing.T) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.MkdirAll("testdata/Foo/BAR", 0755) + + cases := []struct { + name string + conflictFree bool + }{ + // Exists + {"Foo", true}, + {"Foo/BAR", true}, + {"Foo/BAR/baz", true}, + // Doesn't exist + {"bar", true}, + {"Foo/baz", true}, + } + + for _, tc := range cases { + nativeName := filepath.FromSlash(tc.name) + if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { + t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) + } + } +} + +func TestCheckNameConflictCasing(t *testing.T) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.MkdirAll("testdata/Foo/BAR/baz", 0755) + // check if the file system is case-sensitive + if _, err := os.Lstat("testdata/foo"); err != nil { + t.Skip("pointless test") + return + } + + cases := []struct { + name string + conflictFree bool + }{ + // Conflicts + {"foo", false}, + {"foo/BAR", false}, + {"Foo/bar", false}, + {"Foo/BAR/BAZ", false}, + } + + for _, tc := range cases { + nativeName := filepath.FromSlash(tc.name) + if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { + t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) + } + } +} diff --git a/lib/osutil/name_conflict_unix.go b/lib/osutil/name_conflict_unix.go new file mode 100644 index 00000000000..dff645cb60d --- /dev/null +++ b/lib/osutil/name_conflict_unix.go @@ -0,0 +1,57 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows + +package osutil + +import ( + "errors" + "os" + "path/filepath" + "syscall" +) + +// FindRealFileName returns the real name of the last path component of name. +// Base and name must both be clean and name must be relative to base. +// If the last path component of name doesn't exist "" is returned. +func FindRealFileName(base, name string) (string, error) { + // Conflicts can be caused by different casing (e.g. foo and FOO). + info, err := os.Lstat(filepath.Join(base, name)) + if os.IsNotExist(err) { + return "", nil + } + if err != nil { + return "", err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", errors.New("Not a syscall.Stat_t") + } + targetIno := stat.Ino + fd, err := os.Open(filepath.Join(base, filepath.Dir(name))) + if err != nil { + // possible race condition + return "", err + } + defer fd.Close() + for { + infos, err := fd.Readdir(256) + if err != nil { + // possible race condition + return "", err + } + for _, info := range infos { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", errors.New("Not a syscall.Stat_t") + } + if stat.Ino == targetIno { + return info.Name(), nil + } + } + } +} diff --git a/lib/osutil/name_conflict_windows.go b/lib/osutil/name_conflict_windows.go index 1014c92eb9c..025a487c226 100644 --- a/lib/osutil/name_conflict_windows.go +++ b/lib/osutil/name_conflict_windows.go @@ -9,40 +9,29 @@ package osutil import ( - "os" "path/filepath" - "strings" "syscall" ) -// CheckNameConflict returns true if every path component of name up to and -// including filepath.Join(base, name) doesn't conflict with any existing -// files or folders with different names. Base and name must both be clean and -// name must be relative to base. -func CheckNameConflict(base, name string) bool { +// FindRealFileName returns the real name of the last path component of name. +// Base and name must both be clean and name must be relative to base. +// If the last path component of name doesn't exist "" is returned. +func FindRealFileName(base, name string) (string, error) { // Conflicts can be caused by different casing (e.g. foo and FOO) or // by the use of short names (e.g. foo.barbaz and FOO~1.BAR). - path := base - parts := strings.Split(name, string(os.PathSeparator)) - for _, part := range parts { - path = filepath.Join(path, part) - pathp, err := syscall.UTF16PtrFromString(path) - if err != nil { - return false - } - var data syscall.Win32finddata - handle, err := syscall.FindFirstFile(pathp, &data) - if err == syscall.ERROR_FILE_NOT_FOUND { - return true - } - if err != nil { - return false - } - syscall.FindClose(handle) - fileName := syscall.UTF16ToString(data.FileName[:]) - if part != fileName { - return false - } + path := filepath.Join(base, name) + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return "", err } - return true + var data syscall.Win32finddata + handle, err := syscall.FindFirstFile(pathp, &data) + if err == syscall.ERROR_FILE_NOT_FOUND { + return "", nil + } + if err != nil { + return "", err + } + syscall.FindClose(handle) + return syscall.UTF16ToString(data.FileName[:]), nil } diff --git a/lib/osutil/name_conflict_windows_test.go b/lib/osutil/name_conflict_windows_test.go index f8e6aacef7f..e85821ff566 100644 --- a/lib/osutil/name_conflict_windows_test.go +++ b/lib/osutil/name_conflict_windows_test.go @@ -19,42 +19,6 @@ import ( "github.com/syncthing/syncthing/lib/osutil" ) -func TestCheckNameConflictCasing(t *testing.T) { - os.RemoveAll("testdata") - defer os.RemoveAll("testdata") - os.MkdirAll("testdata/Foo/BAR/baz", 0755) - // check if the file system is case-sensitive - if _, err := os.Lstat("testdata/foo"); err != nil { - t.Skip("pointless test") - return - } - - cases := []struct { - name string - conflictFree bool - }{ - // Exists - {"Foo", true}, - {"Foo/BAR", true}, - {"Foo/BAR/baz", true}, - // Doesn't exist - {"bar", true}, - {"Foo/baz", true}, - // Conflicts - {"foo", false}, - {"foo/BAR", false}, - {"Foo/bar", false}, - {"Foo/BAR/BAZ", false}, - } - - for _, tc := range cases { - nativeName := filepath.FromSlash(tc.name) - if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { - t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) - } - } -} - func TestCheckNameConflictShortName(t *testing.T) { os.RemoveAll("testdata") defer os.RemoveAll("testdata") @@ -91,7 +55,6 @@ func TestCheckNameConflictShortName(t *testing.T) { {"foobarbaz", true}, {"foobarbaz/qux", true}, // Doesn't exist - {"foo", true}, {"foobarbaz/quux", true}, // Conflicts {shortName, false}, From d95b0aef3d9550c99c60c3b04f4df52d7ee369c9 Mon Sep 17 00:00:00 2001 From: Unrud Date: Tue, 27 Dec 2016 15:52:31 +0100 Subject: [PATCH 11/13] Allow deletion updates for conflicting or Symlink-traversing files TODO: Rebase with #3840 --- lib/model/rwfolder.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index f7c354d13ef..7655196a8e8 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -429,17 +429,29 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int { buckets := map[string][]protocol.FileInfo{} for _, fi := range processDirectly { + handlePathError := func(err error) { + if fi.IsDeleted() { + if fi.IsDirectory() { + f.dbUpdates <- dbUpdateJob{fi, dbUpdateDeleteDir} + } else { + f.dbUpdates <- dbUpdateJob{fi, dbUpdateDeleteFile} + } + } else { + f.newError(fi.Name, err) + } + } + // Verify that the thing we are handling lives inside a directory, // and not a symlink or empty space. if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil { - f.newError(fi.Name, err) + handlePathError(err) continue } // Verify that we handle the right thing and not something whose name // collides. if !osutil.CheckNameConflict(f.dir, fi.Name) { - f.newError(fi.Name, errNameConflict) + handlePathError(errNameConflict) continue } @@ -1588,7 +1600,10 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() { // fsyncing symlinks is only supported by MacOS, ignore } if job.jobType != dbUpdateShortcutFile { - changedDirs = append(changedDirs, filepath.Dir(filepath.Join(f.dir, job.file.Name))) + err := osutil.TraversesSymlink(f.dir, filepath.Dir(job.file.Name)) + if err == nil && osutil.CheckNameConflict(f.dir, job.file.Name) { + changedDirs = append(changedDirs, filepath.Dir(filepath.Join(f.dir, job.file.Name))) + } } } if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) { From caa0dd671f34b243e1144e773cecf63118788e3f Mon Sep 17 00:00:00 2001 From: Unrud Date: Thu, 19 Jan 2017 12:17:22 +0100 Subject: [PATCH 12/13] Track version number for each folder in the db --- cmd/syncthing/main.go | 6 ++++ lib/config/config.go | 10 +++++- lib/config/testdata/v18.xml | 15 +++++++++ lib/db/leveldb.go | 1 + lib/db/leveldb_dbinstance.go | 59 ++++++++++++++++++++++++++++++++++++ lib/db/set.go | 29 ++++++++++++++++-- lib/model/model.go | 9 ++++++ 7 files changed, 125 insertions(+), 4 deletions(-) create mode 100644 lib/config/testdata/v18.xml diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index f5fbfc3d56f..6636ad725a6 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -721,6 +721,12 @@ func syncthingMain(runtimeOptions RuntimeOptions) { ldb.DropDeltaIndexIDs() } + if cfg.RawCopy().OriginalVersion <= 17 { + // The config version 17->18 migration is about tracking folder versions + // in the database + ldb.AddFolderVersions() + } + m := model.NewModel(cfg, myID, myDeviceName(cfg), "syncthing", Version, ldb, protectedFiles) if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 { diff --git a/lib/config/config.go b/lib/config/config.go index fac91683b25..652c2bb7be5 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -26,7 +26,7 @@ import ( const ( OldestHandledVersion = 10 - CurrentVersion = 17 + CurrentVersion = 18 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) @@ -257,6 +257,9 @@ func (cfg *Configuration) clean() error { if cfg.Version == 16 { convertV16V17(cfg) } + if cfg.Version == 17 { + convertV17V18(cfg) + } // Build a list of available devices existingDevices := make(map[protocol.DeviceID]bool) @@ -338,6 +341,11 @@ func convertV16V17(cfg *Configuration) { cfg.Version = 17 } +func convertV17V18(cfg *Configuration) { + // Triggers a database tweak + cfg.Version = 18 +} + func convertV13V14(cfg *Configuration) { // Not using the ignore cache is the new default. Disable it on existing // configurations. diff --git a/lib/config/testdata/v18.xml b/lib/config/testdata/v18.xml new file mode 100644 index 00000000000..4dc56e66e41 --- /dev/null +++ b/lib/config/testdata/v18.xml @@ -0,0 +1,15 @@ + + + + + 1 + -1 + true + + +
tcp://a
+ + +
tcp://b
+ + diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index c4f3abe4a72..5fc19f9bb33 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -25,6 +25,7 @@ const ( KeyTypeFolderIdx KeyTypeDeviceIdx KeyTypeIndexID + KeyTypeFolderVersion ) func (l VersionList) String() string { diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index ffc3d5d3045..94e062ccafc 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -707,6 +707,32 @@ func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) { } } +func (db *Instance) getFolderVersion(folder []byte) uint64 { + key := db.folderVersionKey(folder) + cur, err := db.Get(key, nil) + if err != nil { + return 0 + } + if len(cur) != 8 { + return 0 + } + return binary.BigEndian.Uint64(cur) +} + +func (db *Instance) setFolderVersion(folder []byte, version uint64) { + key := db.folderVersionKey(folder) + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, version) + if err := db.Put(key, bs, nil); err != nil { + panic("storing folder version: " + err.Error()) + } +} + +func (db *Instance) dropFolderVersion(folder []byte) { + key := db.folderVersionKey(folder) + db.dropPrefix(key) +} + func (db *Instance) indexIDKey(device, folder []byte) []byte { k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen) k[0] = KeyTypeIndexID @@ -722,12 +748,45 @@ func (db *Instance) mtimesKey(folder []byte) []byte { return prefix } +func (db *Instance) folderVersionKey(folder []byte) []byte { + k := make([]byte, keyPrefixLen+keyFolderLen) + k[0] = KeyTypeFolderVersion + binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder)) + return k +} + // DropDeltaIndexIDs removes all index IDs from the database. This will // cause a full index transmission on the next connection. func (db *Instance) DropDeltaIndexIDs() { db.dropPrefix([]byte{KeyTypeIndexID}) } +func (db *Instance) AddFolderVersions() { + t := db.newReadOnlyTransaction() + defer t.close() + + // Set folder version of all found folders to 1 + done := make(map[string]bool) + k := make([]byte, keyPrefixLen+keyFolderLen) + k[0] = KeyTypeFolderVersion + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, 1) + dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil) + for dbi.Next() { + keyFolder := dbi.Key()[keyPrefixLen : keyPrefixLen+keyFolderLen] + if !done[string(keyFolder)] { + copy(k[keyPrefixLen:], keyFolder) + if _, err := db.Get(k, nil); err == leveldb.ErrNotFound { + db.Put(k, bs, nil) + } else if err != nil { + panic(err) + } + done[string(keyFolder)] = true + } + } + dbi.Release() +} + func (db *Instance) dropMtimes(folder []byte) { db.dropPrefix(db.mtimesKey(folder)) } diff --git a/lib/db/set.go b/lib/db/set.go index f432e1b9893..fa636283404 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -22,9 +22,14 @@ import ( "github.com/syncthing/syncthing/lib/sync" ) +const ( + CurrentFolderVersion = 1 +) + type FileSet struct { sequence int64 // Our local sequence number folder string + version uint64 db *Instance blockmap *BlockMap localSize sizeTracker @@ -117,6 +122,7 @@ func NewFileSet(folder string, db *Instance) *FileSet { var s = FileSet{ remoteSequence: make(map[protocol.DeviceID]int64), folder: folder, + version: db.getFolderVersion([]byte(folder)), db: db, blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))), updateMutex: sync.NewMutex(), @@ -171,12 +177,15 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) { } func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) { - l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs)) - normalizeFilenames(fs) - s.updateMutex.Lock() defer s.updateMutex.Unlock() + s.updateLocked(device, fs) +} + +func (s *FileSet) updateLocked(device protocol.DeviceID, fs []protocol.FileInfo) { + l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs)) + normalizeFilenames(fs) if device == protocol.LocalDeviceID { discards := make([]protocol.FileInfo, 0, len(fs)) updates := make([]protocol.FileInfo, 0, len(fs)) @@ -318,6 +327,19 @@ func (s *FileSet) ListDevices() []protocol.DeviceID { return devices } +func (s *FileSet) UpdateFolderVersion(checkFolderHealth func() error, checkNameConflict func(string) bool) error { + if s.version == CurrentFolderVersion { + return nil + } + s.updateMutex.Lock() + defer s.updateMutex.Unlock() + defer s.db.setFolderVersion([]byte(s.folder), s.version) + if s.version == 0 { + s.version = CurrentFolderVersion + } + return nil +} + // maxSequence returns the highest of the Sequence numbers found in // the given slice of FileInfos. This should really be the Sequence of // the last item, but Syncthing v0.14.0 and other implementations may not @@ -337,6 +359,7 @@ func maxSequence(fs []protocol.FileInfo) int64 { func DropFolder(db *Instance, folder string) { db.dropFolder([]byte(folder)) db.dropMtimes([]byte(folder)) + db.dropFolderVersion([]byte(folder)) bm := &BlockMap{ db: db, folder: db.folderIdx.ID([]byte(folder)), diff --git a/lib/model/model.go b/lib/model/model.go index c487f6a454c..b95d351d035 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -1789,6 +1789,15 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error return err } + if err := fs.UpdateFolderVersion(func() error { + return m.CheckFolderHealth(folder) + }, func(name string) bool { + return osutil.CheckNameConflict(folderCfg.Path(), name) + }); err != nil { + l.Infof("Stopping folder %s mid-update due to folder error: %s", folderCfg.Description(), err) + return err + } + // Clean the list of subitems to ensure that we start at a known // directory, and don't scan subdirectories of things we've already // scanned. From 98f44bd9db561f4e374dbeab3b016a8015bb832a Mon Sep 17 00:00:00 2001 From: Unrud Date: Thu, 19 Jan 2017 12:18:48 +0100 Subject: [PATCH 13/13] Mark conflicting files as invalid (Update folder version to 2) Prevent deletion of files with conflicting names in the cluster --- lib/db/set.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/lib/db/set.go b/lib/db/set.go index fa636283404..cf0a6a9e23b 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -23,7 +23,7 @@ import ( ) const ( - CurrentFolderVersion = 1 + CurrentFolderVersion = 2 ) type FileSet struct { @@ -337,6 +337,59 @@ func (s *FileSet) UpdateFolderVersion(checkFolderHealth func() error, checkNameC if s.version == 0 { s.version = CurrentFolderVersion } + if s.version == 1 { + if err := s.convertV1V2(checkFolderHealth, checkNameConflict); err != nil { + return err + } + } + return nil +} + +func (s *FileSet) convertV1V2(checkFolderHealth func() error, checkNameConflict func(string) bool) error { + batchSizeFiles := 100 + batch := make([]protocol.FileInfo, 0, batchSizeFiles) + var iterError error + s.WithHaveTruncated(protocol.LocalDeviceID, func(fi FileIntf) bool { + f := fi.(FileInfoTruncated) + if len(batch) == batchSizeFiles { + if err := checkFolderHealth(); err != nil { + iterError = err + return false + } + s.updateLocked(protocol.LocalDeviceID, batch) + batch = batch[:0] + } + + if !f.IsInvalid() && !f.IsDeleted() && !checkNameConflict(f.Name) { + // Mark conflicting files as invalid to prevent deletion on other devices + l.Debugln("setting invalid bit on conflicting", f) + nf := protocol.FileInfo{ + Name: f.Name, + Type: f.Type, + Size: f.Size, + ModifiedS: f.ModifiedS, + ModifiedNs: f.ModifiedNs, + Permissions: f.Permissions, + NoPermissions: f.NoPermissions, + Invalid: true, + Version: f.Version, // The file is still the same, so don't bump version + } + batch = append(batch, nf) + } + return true + }) + + if iterError != nil { + return iterError + } + + if err := checkFolderHealth(); err != nil { + return err + } else if len(batch) > 0 { + s.updateLocked(protocol.LocalDeviceID, batch) + } + + s.version = 2 return nil }