diff --git a/cmd/syncthing/main.go b/cmd/syncthing/main.go index f5fbfc3d56f..6636ad725a6 100644 --- a/cmd/syncthing/main.go +++ b/cmd/syncthing/main.go @@ -721,6 +721,12 @@ func syncthingMain(runtimeOptions RuntimeOptions) { ldb.DropDeltaIndexIDs() } + if cfg.RawCopy().OriginalVersion <= 17 { + // The config version 17->18 migration is about tracking folder versions + // in the database + ldb.AddFolderVersions() + } + m := model.NewModel(cfg, myID, myDeviceName(cfg), "syncthing", Version, ldb, protectedFiles) if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 { diff --git a/lib/config/config.go b/lib/config/config.go index fac91683b25..652c2bb7be5 100644 --- a/lib/config/config.go +++ b/lib/config/config.go @@ -26,7 +26,7 @@ import ( const ( OldestHandledVersion = 10 - CurrentVersion = 17 + CurrentVersion = 18 MaxRescanIntervalS = 365 * 24 * 60 * 60 ) @@ -257,6 +257,9 @@ func (cfg *Configuration) clean() error { if cfg.Version == 16 { convertV16V17(cfg) } + if cfg.Version == 17 { + convertV17V18(cfg) + } // Build a list of available devices existingDevices := make(map[protocol.DeviceID]bool) @@ -338,6 +341,11 @@ func convertV16V17(cfg *Configuration) { cfg.Version = 17 } +func convertV17V18(cfg *Configuration) { + // Triggers a database tweak + cfg.Version = 18 +} + func convertV13V14(cfg *Configuration) { // Not using the ignore cache is the new default. Disable it on existing // configurations. diff --git a/lib/config/testdata/v18.xml b/lib/config/testdata/v18.xml new file mode 100644 index 00000000000..4dc56e66e41 --- /dev/null +++ b/lib/config/testdata/v18.xml @@ -0,0 +1,15 @@ + + + + + 1 + -1 + true + + +
tcp://a
+ + +
tcp://b
+ + diff --git a/lib/db/leveldb.go b/lib/db/leveldb.go index c4f3abe4a72..5fc19f9bb33 100644 --- a/lib/db/leveldb.go +++ b/lib/db/leveldb.go @@ -25,6 +25,7 @@ const ( KeyTypeFolderIdx KeyTypeDeviceIdx KeyTypeIndexID + KeyTypeFolderVersion ) func (l VersionList) String() string { diff --git a/lib/db/leveldb_dbinstance.go b/lib/db/leveldb_dbinstance.go index ffc3d5d3045..94e062ccafc 100644 --- a/lib/db/leveldb_dbinstance.go +++ b/lib/db/leveldb_dbinstance.go @@ -707,6 +707,32 @@ func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) { } } +func (db *Instance) getFolderVersion(folder []byte) uint64 { + key := db.folderVersionKey(folder) + cur, err := db.Get(key, nil) + if err != nil { + return 0 + } + if len(cur) != 8 { + return 0 + } + return binary.BigEndian.Uint64(cur) +} + +func (db *Instance) setFolderVersion(folder []byte, version uint64) { + key := db.folderVersionKey(folder) + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, version) + if err := db.Put(key, bs, nil); err != nil { + panic("storing folder version: " + err.Error()) + } +} + +func (db *Instance) dropFolderVersion(folder []byte) { + key := db.folderVersionKey(folder) + db.dropPrefix(key) +} + func (db *Instance) indexIDKey(device, folder []byte) []byte { k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen) k[0] = KeyTypeIndexID @@ -722,12 +748,45 @@ func (db *Instance) mtimesKey(folder []byte) []byte { return prefix } +func (db *Instance) folderVersionKey(folder []byte) []byte { + k := make([]byte, keyPrefixLen+keyFolderLen) + k[0] = KeyTypeFolderVersion + binary.BigEndian.PutUint32(k[keyPrefixLen:], db.folderIdx.ID(folder)) + return k +} + // DropDeltaIndexIDs removes all index IDs from the database. This will // cause a full index transmission on the next connection. func (db *Instance) DropDeltaIndexIDs() { db.dropPrefix([]byte{KeyTypeIndexID}) } +func (db *Instance) AddFolderVersions() { + t := db.newReadOnlyTransaction() + defer t.close() + + // Set folder version of all found folders to 1 + done := make(map[string]bool) + k := make([]byte, keyPrefixLen+keyFolderLen) + k[0] = KeyTypeFolderVersion + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, 1) + dbi := t.NewIterator(util.BytesPrefix([]byte{KeyTypeDevice}), nil) + for dbi.Next() { + keyFolder := dbi.Key()[keyPrefixLen : keyPrefixLen+keyFolderLen] + if !done[string(keyFolder)] { + copy(k[keyPrefixLen:], keyFolder) + if _, err := db.Get(k, nil); err == leveldb.ErrNotFound { + db.Put(k, bs, nil) + } else if err != nil { + panic(err) + } + done[string(keyFolder)] = true + } + } + dbi.Release() +} + func (db *Instance) dropMtimes(folder []byte) { db.dropPrefix(db.mtimesKey(folder)) } diff --git a/lib/db/set.go b/lib/db/set.go index f432e1b9893..cf0a6a9e23b 100644 --- a/lib/db/set.go +++ b/lib/db/set.go @@ -22,9 +22,14 @@ import ( "github.com/syncthing/syncthing/lib/sync" ) +const ( + CurrentFolderVersion = 2 +) + type FileSet struct { sequence int64 // Our local sequence number folder string + version uint64 db *Instance blockmap *BlockMap localSize sizeTracker @@ -117,6 +122,7 @@ func NewFileSet(folder string, db *Instance) *FileSet { var s = FileSet{ remoteSequence: make(map[protocol.DeviceID]int64), folder: folder, + version: db.getFolderVersion([]byte(folder)), db: db, blockmap: NewBlockMap(db, db.folderIdx.ID([]byte(folder))), updateMutex: sync.NewMutex(), @@ -171,12 +177,15 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) { } func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) { - l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs)) - normalizeFilenames(fs) - s.updateMutex.Lock() defer s.updateMutex.Unlock() + s.updateLocked(device, fs) +} + +func (s *FileSet) updateLocked(device protocol.DeviceID, fs []protocol.FileInfo) { + l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs)) + normalizeFilenames(fs) if device == protocol.LocalDeviceID { discards := make([]protocol.FileInfo, 0, len(fs)) updates := make([]protocol.FileInfo, 0, len(fs)) @@ -318,6 +327,72 @@ func (s *FileSet) ListDevices() []protocol.DeviceID { return devices } +func (s *FileSet) UpdateFolderVersion(checkFolderHealth func() error, checkNameConflict func(string) bool) error { + if s.version == CurrentFolderVersion { + return nil + } + s.updateMutex.Lock() + defer s.updateMutex.Unlock() + defer s.db.setFolderVersion([]byte(s.folder), s.version) + if s.version == 0 { + s.version = CurrentFolderVersion + } + if s.version == 1 { + if err := s.convertV1V2(checkFolderHealth, checkNameConflict); err != nil { + return err + } + } + return nil +} + +func (s *FileSet) convertV1V2(checkFolderHealth func() error, checkNameConflict func(string) bool) error { + batchSizeFiles := 100 + batch := make([]protocol.FileInfo, 0, batchSizeFiles) + var iterError error + s.WithHaveTruncated(protocol.LocalDeviceID, func(fi FileIntf) bool { + f := fi.(FileInfoTruncated) + if len(batch) == batchSizeFiles { + if err := checkFolderHealth(); err != nil { + iterError = err + return false + } + s.updateLocked(protocol.LocalDeviceID, batch) + batch = batch[:0] + } + + if !f.IsInvalid() && !f.IsDeleted() && !checkNameConflict(f.Name) { + // Mark conflicting files as invalid to prevent deletion on other devices + l.Debugln("setting invalid bit on conflicting", f) + nf := protocol.FileInfo{ + Name: f.Name, + Type: f.Type, + Size: f.Size, + ModifiedS: f.ModifiedS, + ModifiedNs: f.ModifiedNs, + Permissions: f.Permissions, + NoPermissions: f.NoPermissions, + Invalid: true, + Version: f.Version, // The file is still the same, so don't bump version + } + batch = append(batch, nf) + } + return true + }) + + if iterError != nil { + return iterError + } + + if err := checkFolderHealth(); err != nil { + return err + } else if len(batch) > 0 { + s.updateLocked(protocol.LocalDeviceID, batch) + } + + s.version = 2 + return nil +} + // maxSequence returns the highest of the Sequence numbers found in // the given slice of FileInfos. This should really be the Sequence of // the last item, but Syncthing v0.14.0 and other implementations may not @@ -337,6 +412,7 @@ func maxSequence(fs []protocol.FileInfo) int64 { func DropFolder(db *Instance, folder string) { db.dropFolder([]byte(folder)) db.dropMtimes([]byte(folder)) + db.dropFolderVersion([]byte(folder)) bm := &BlockMap{ db: db, folder: db.folderIdx.ID([]byte(folder)), diff --git a/lib/model/model.go b/lib/model/model.go index b06743702e9..b95d351d035 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -121,6 +121,7 @@ var ( errDevicePaused = errors.New("device is paused") errDeviceIgnored = errors.New("device is ignored") errNotRelative = errors.New("not a relative path") + errNameConflict = errors.New("filename collides with existing file") ) // NewModel creates and starts a new model. The model starts in read-only mode, @@ -1163,6 +1164,11 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset return protocol.ErrNoSuchFile } + if !osutil.CheckNameConflict(folderPath, name) { + l.Debugf("%v REQ(in) for file not in dir: %s: %q / %q o=%d s=%d", m, deviceID, folder, name, offset, len(buf)) + return protocol.ErrNoSuchFile + } + // Only check temp files if the flag is set, and if we are set to advertise // the temp indexes. if fromTemporary && !folderCfg.DisableTempIndexes { @@ -1783,6 +1789,15 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error return err } + if err := fs.UpdateFolderVersion(func() error { + return m.CheckFolderHealth(folder) + }, func(name string) bool { + return osutil.CheckNameConflict(folderCfg.Path(), name) + }); err != nil { + l.Infof("Stopping folder %s mid-update due to folder error: %s", folderCfg.Description(), err) + return err + } + // Clean the list of subitems to ensure that we start at a known // directory, and don't scan subdirectories of things we've already // scanned. @@ -1899,7 +1914,12 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error // The file is valid and not deleted. Lets check if it's // still here. - if _, err := mtimefs.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil { + var exists bool + if err := osutil.TraversesSymlink(folderCfg.Path(), filepath.Dir(f.Name)); err != nil { + exists = false + } else if !osutil.CheckNameConflict(folderCfg.Path(), f.Name) { + exists = false + } else if _, err := mtimefs.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil { // We don't specifically verify that the error is // os.IsNotExist because there is a corner case when a // directory is suddenly transformed into a file. When that @@ -1907,6 +1927,11 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error // file) are deleted but will return a confusing error ("not a // directory") when we try to Lstat() them. + exists = false + } else { + exists = true + } + if !exists { nf := protocol.FileInfo{ Name: f.Name, Type: f.Type, diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index ca4b85d0ced..7655196a8e8 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -429,10 +429,29 @@ func (f *sendReceiveFolder) pullerIteration(ignores *ignore.Matcher) int { buckets := map[string][]protocol.FileInfo{} for _, fi := range processDirectly { + handlePathError := func(err error) { + if fi.IsDeleted() { + if fi.IsDirectory() { + f.dbUpdates <- dbUpdateJob{fi, dbUpdateDeleteDir} + } else { + f.dbUpdates <- dbUpdateJob{fi, dbUpdateDeleteFile} + } + } else { + f.newError(fi.Name, err) + } + } + // Verify that the thing we are handling lives inside a directory, // and not a symlink or empty space. if err := osutil.TraversesSymlink(f.dir, filepath.Dir(fi.Name)); err != nil { - f.newError(fi.Name, err) + handlePathError(err) + continue + } + + // Verify that we handle the right thing and not something whose name + // collides. + if !osutil.CheckNameConflict(f.dir, fi.Name) { + handlePathError(errNameConflict) continue } @@ -525,6 +544,13 @@ nextFile: continue } + // Verify that we handle the right thing and not something whose name + // collides. + if !osutil.CheckNameConflict(f.dir, fi.Name) { + f.newError(fi.Name, errNameConflict) + continue + } + // Check our list of files to be removed for a match, in which case // we can just do a rename instead. key := string(fi.Blocks[0].Hash) @@ -1283,6 +1309,16 @@ func (f *sendReceiveFolder) copierRoutine(in <-chan copyBlocksState, pullChan ch if err != nil { return false } + // The following checks are racy + if err := osutil.TraversesSymlink(folderRoots[folder], filepath.Dir(file)); err != nil { + return false + } + if !osutil.CheckNameConflict(folderRoots[folder], file) { + return false + } + if info, err := osutil.Lstat(inFile); err != nil || !info.Mode().IsRegular() { + return false + } fd, err := os.Open(inFile) if err != nil { return false @@ -1564,7 +1600,10 @@ func (f *sendReceiveFolder) dbUpdaterRoutine() { // fsyncing symlinks is only supported by MacOS, ignore } if job.jobType != dbUpdateShortcutFile { - changedDirs = append(changedDirs, filepath.Dir(filepath.Join(f.dir, job.file.Name))) + err := osutil.TraversesSymlink(f.dir, filepath.Dir(job.file.Name)) + if err == nil && osutil.CheckNameConflict(f.dir, job.file.Name) { + changedDirs = append(changedDirs, filepath.Dir(filepath.Join(f.dir, job.file.Name))) + } } } if job.file.IsInvalid() || (job.file.IsDirectory() && !job.file.IsSymlink()) { diff --git a/lib/osutil/name_conflict.go b/lib/osutil/name_conflict.go new file mode 100644 index 00000000000..8341cf78eef --- /dev/null +++ b/lib/osutil/name_conflict.go @@ -0,0 +1,39 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package osutil + +import ( + "os" + "path/filepath" + "strings" +) + +// CheckNameConflict returns true if every path component of name up to and +// including filepath.Join(base, name) doesn't conflict with any existing +// files or folders with different names. Base and name must both be clean and +// name must be relative to base. +func CheckNameConflict(base, name string) bool { + // Conflicts depend on the OS and file system. + subname := "." + parts := strings.Split(name, string(os.PathSeparator)) + for _, part := range parts { + subname = filepath.Join(subname, part) + fileName, err := FindRealFileName(base, subname) + if err != nil { + return false + } + if fileName == "" { + // doesn't exist + return true + } + if fileName != part { + // conflict + return false + } + } + return true +} diff --git a/lib/osutil/name_conflict_test.go b/lib/osutil/name_conflict_test.go new file mode 100644 index 00000000000..1ae70659dfa --- /dev/null +++ b/lib/osutil/name_conflict_test.go @@ -0,0 +1,70 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package osutil_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/syncthing/syncthing/lib/osutil" +) + +func TestCheckNameConflict(t *testing.T) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.MkdirAll("testdata/Foo/BAR", 0755) + + cases := []struct { + name string + conflictFree bool + }{ + // Exists + {"Foo", true}, + {"Foo/BAR", true}, + {"Foo/BAR/baz", true}, + // Doesn't exist + {"bar", true}, + {"Foo/baz", true}, + } + + for _, tc := range cases { + nativeName := filepath.FromSlash(tc.name) + if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { + t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) + } + } +} + +func TestCheckNameConflictCasing(t *testing.T) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.MkdirAll("testdata/Foo/BAR/baz", 0755) + // check if the file system is case-sensitive + if _, err := os.Lstat("testdata/foo"); err != nil { + t.Skip("pointless test") + return + } + + cases := []struct { + name string + conflictFree bool + }{ + // Conflicts + {"foo", false}, + {"foo/BAR", false}, + {"Foo/bar", false}, + {"Foo/BAR/BAZ", false}, + } + + for _, tc := range cases { + nativeName := filepath.FromSlash(tc.name) + if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { + t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) + } + } +} diff --git a/lib/osutil/name_conflict_unix.go b/lib/osutil/name_conflict_unix.go new file mode 100644 index 00000000000..dff645cb60d --- /dev/null +++ b/lib/osutil/name_conflict_unix.go @@ -0,0 +1,57 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build !windows + +package osutil + +import ( + "errors" + "os" + "path/filepath" + "syscall" +) + +// FindRealFileName returns the real name of the last path component of name. +// Base and name must both be clean and name must be relative to base. +// If the last path component of name doesn't exist "" is returned. +func FindRealFileName(base, name string) (string, error) { + // Conflicts can be caused by different casing (e.g. foo and FOO). + info, err := os.Lstat(filepath.Join(base, name)) + if os.IsNotExist(err) { + return "", nil + } + if err != nil { + return "", err + } + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", errors.New("Not a syscall.Stat_t") + } + targetIno := stat.Ino + fd, err := os.Open(filepath.Join(base, filepath.Dir(name))) + if err != nil { + // possible race condition + return "", err + } + defer fd.Close() + for { + infos, err := fd.Readdir(256) + if err != nil { + // possible race condition + return "", err + } + for _, info := range infos { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return "", errors.New("Not a syscall.Stat_t") + } + if stat.Ino == targetIno { + return info.Name(), nil + } + } + } +} diff --git a/lib/osutil/name_conflict_windows.go b/lib/osutil/name_conflict_windows.go new file mode 100644 index 00000000000..025a487c226 --- /dev/null +++ b/lib/osutil/name_conflict_windows.go @@ -0,0 +1,37 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows + +package osutil + +import ( + "path/filepath" + "syscall" +) + +// FindRealFileName returns the real name of the last path component of name. +// Base and name must both be clean and name must be relative to base. +// If the last path component of name doesn't exist "" is returned. +func FindRealFileName(base, name string) (string, error) { + // Conflicts can be caused by different casing (e.g. foo and FOO) or + // by the use of short names (e.g. foo.barbaz and FOO~1.BAR). + path := filepath.Join(base, name) + pathp, err := syscall.UTF16PtrFromString(path) + if err != nil { + return "", err + } + var data syscall.Win32finddata + handle, err := syscall.FindFirstFile(pathp, &data) + if err == syscall.ERROR_FILE_NOT_FOUND { + return "", nil + } + if err != nil { + return "", err + } + syscall.FindClose(handle) + return syscall.UTF16ToString(data.FileName[:]), nil +} diff --git a/lib/osutil/name_conflict_windows_test.go b/lib/osutil/name_conflict_windows_test.go new file mode 100644 index 00000000000..e85821ff566 --- /dev/null +++ b/lib/osutil/name_conflict_windows_test.go @@ -0,0 +1,71 @@ +// Copyright (C) 2016 The Syncthing Authors. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +// +build windows + +package osutil_test + +import ( + "os" + "path" + "path/filepath" + "syscall" + "testing" + "unsafe" + + "github.com/syncthing/syncthing/lib/osutil" +) + +func TestCheckNameConflictShortName(t *testing.T) { + os.RemoveAll("testdata") + defer os.RemoveAll("testdata") + os.MkdirAll("testdata/foobarbaz/qux", 0755) + ppath, err := syscall.UTF16PtrFromString("testdata/foobarbaz") + if err != nil { + t.Fatal("unexpected error", err) + } + // check if the file system supports short names + bufferSize, err := syscall.GetShortPathName(ppath, nil, 0) + if err != nil { + t.Skip("pointless test") + return + } + + // get the short name + buffer := make([]uint16, bufferSize) + length, err := syscall.GetShortPathName(ppath, + (*uint16)(unsafe.Pointer(&buffer[0])), bufferSize) + if err != nil { + t.Fatal("unexpected error", err) + } + // on success length doesn't contain the terminating null character + if bufferSize != length+1 { + t.Fatal("length of short name changed") + } + shortName := filepath.Base(syscall.UTF16ToString(buffer)) + + cases := []struct { + name string + conflictFree bool + }{ + // Exists + {"foobarbaz", true}, + {"foobarbaz/qux", true}, + // Doesn't exist + {"foobarbaz/quux", true}, + // Conflicts + {shortName, false}, + {path.Join(shortName, "qux"), false}, + {path.Join(shortName, "quux"), false}, + } + + for _, tc := range cases { + nativeName := filepath.FromSlash(tc.name) + if res := osutil.CheckNameConflict("testdata", nativeName); res != tc.conflictFree { + t.Errorf("CheckNameConflict(%q) = %v, should be %v", tc.name, res, tc.conflictFree) + } + } +} diff --git a/lib/scanner/walk.go b/lib/scanner/walk.go index 5c1e2649cf4..83e7d987d49 100644 --- a/lib/scanner/walk.go +++ b/lib/scanner/walk.go @@ -120,6 +120,14 @@ func (w *walker) walk() (chan protocol.FileInfo, error) { filepath.Walk(w.Dir, hashFiles) } else { for _, sub := range w.Subs { + if err := osutil.TraversesSymlink(w.Dir, filepath.Dir(sub)); err != nil { + l.Infoln("Skipping sub path that traverses symlink", w.Dir, sub) + continue + } + if !osutil.CheckNameConflict(w.Dir, sub) { + l.Infoln("Skipping sub path that collides", w.Dir, sub) + continue + } filepath.Walk(filepath.Join(w.Dir, sub), hashFiles) } }