Skip to content

Commit

Permalink
Add pruner
Browse files Browse the repository at this point in the history
  • Loading branch information
AudriusButkevicius committed Apr 27, 2015
1 parent 15d9a23 commit 47804eb
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 18 deletions.
22 changes: 16 additions & 6 deletions internal/auto/gui.files.go

Large diffs are not rendered by default.

12 changes: 11 additions & 1 deletion internal/config/config.go
Expand Up @@ -439,7 +439,17 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
// ChangeRequiresRestart returns true if updating the configuration requires a
// complete restart.
func ChangeRequiresRestart(from, to Configuration) bool {
// Adding, removing or changing folders requires restart
// Changing selective state or patterns does not require a restart.
for _, fromf := range from.Folders {
for _, tof := range to.Folders {
if fromf.ID == tof.ID {
fromf.SelectiveEnabled = tof.SelectiveEnabled
fromf.SelectivePatterns = tof.SelectivePatterns
break
}
}
}
// Adding, removing or changing anything else requires restart
if !reflect.DeepEqual(from.Folders, to.Folders) {
return true
}
Expand Down
84 changes: 74 additions & 10 deletions internal/model/model.go
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/syncthing/syncthing/internal/events"
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/pruner"
"github.com/syncthing/syncthing/internal/scanner"
"github.com/syncthing/syncthing/internal/stats"
"github.com/syncthing/syncthing/internal/symlinks"
Expand Down Expand Up @@ -72,6 +73,7 @@ type Model struct {
deviceFolders map[protocol.DeviceID][]string // deviceID -> folders
deviceStatRefs map[protocol.DeviceID]*stats.DeviceStatisticsReference // deviceID -> statsRef
folderIgnores map[string]*ignore.Matcher // folder -> matcher object
folderPruners map[string]*pruner.Pruner // folder -> pruner object
folderRunners map[string]service // folder -> puller or scanner
folderStatRefs map[string]*stats.FolderStatisticsReference // folder -> statsRef
fmut sync.RWMutex // protects the above
Expand Down Expand Up @@ -109,6 +111,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
deviceFolders: make(map[protocol.DeviceID][]string),
deviceStatRefs: make(map[protocol.DeviceID]*stats.DeviceStatisticsReference),
folderIgnores: make(map[string]*ignore.Matcher),
folderPruners: make(map[string]*pruner.Pruner),
folderRunners: make(map[string]service),
folderStatRefs: make(map[string]*stats.FolderStatisticsReference),
protoConn: make(map[protocol.DeviceID]protocol.Connection),
Expand All @@ -122,9 +125,29 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
go m.progressEmitter.Serve()
}

cfg.Subscribe(m)
m.Changed(cfg.Raw())

return m
}

func (m *Model) Changed(cfg config.Configuration) error {
for _, folder := range cfg.Folders {
// Patterns are updated at the point they are set.
_, ok := m.folderPruners[folder.ID]
if !ok && folder.SelectiveEnabled {
m.fmut.Lock()
m.folderPruners[folder.ID] = pruner.New(folder.SelectivePatterns)
m.fmut.Unlock()
} else if ok && !folder.SelectiveEnabled {
m.fmut.Lock()
delete(m.folderPruners, folder.ID)
m.fmut.Unlock()
}
}
return nil
}

// Starts deadlock detector on the models locks which causes panics in case
// the locks cannot be acquired in the given timeout period.
func (m *Model) StartDeadlockDetector(timeout time.Duration) {
Expand Down Expand Up @@ -926,7 +949,7 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
m.fmut.RLock()
for _, folder := range m.deviceFolders[deviceID] {
fs := m.folderFiles[folder]
go sendIndexes(protoConn, folder, fs, m.folderIgnores[folder])
go sendIndexes(protoConn, folder, fs, m.folderIgnores[folder], m.folderPruners[folder])
}
m.fmut.RUnlock()
m.pmut.Unlock()
Expand Down Expand Up @@ -967,7 +990,7 @@ func (m *Model) receivedFile(folder, filename string) {
m.folderStatRef(folder).ReceivedFile(filename)
}

func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) {
func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher, pruner *pruner.Pruner) {
deviceID := conn.ID()
name := conn.Name()
var err error
Expand All @@ -976,23 +999,23 @@ func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignore
l.Debugf("sendIndexes for %s-%s/%q starting", deviceID, name, folder)
}

minLocalVer, err := sendIndexTo(true, 0, conn, folder, fs, ignores)
minLocalVer, err := sendIndexTo(true, 0, conn, folder, fs, ignores, pruner)

for err == nil {
time.Sleep(5 * time.Second)
if fs.LocalVersion(protocol.LocalDeviceID) <= minLocalVer {
continue
}

minLocalVer, err = sendIndexTo(false, minLocalVer, conn, folder, fs, ignores)
minLocalVer, err = sendIndexTo(false, minLocalVer, conn, folder, fs, ignores, pruner)
}

if debug {
l.Debugf("sendIndexes for %s-%s/%q exiting: %v", deviceID, name, folder, err)
}
}

func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) (int64, error) {
func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher, pruner *pruner.Pruner) (int64, error) {
deviceID := conn.ID()
name := conn.Name()
batch := make([]protocol.FileInfo, 0, indexBatchSize)
Expand All @@ -1010,9 +1033,9 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
maxLocalVer = f.LocalVersion
}

if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) || pruner.ShouldSkip(f) {
if debug {
l.Debugln("not sending update for ignored/unsupported symlink", f)
l.Debugln("not sending update for ignored/pruned/unsupported symlink", f)
}
return true
}
Expand Down Expand Up @@ -1167,6 +1190,7 @@ func (m *Model) ScanFolderSubs(folder string, subs []string) error {
fs := m.folderFiles[folder]
folderCfg := m.folderCfgs[folder]
ignores := m.folderIgnores[folder]
pruner := m.folderPruners[folder]
runner, ok := m.folderRunners[folder]
m.fmut.Unlock()

Expand Down Expand Up @@ -1206,6 +1230,7 @@ nextSub:
Dir: folderCfg.Path(),
Subs: subs,
Matcher: ignores,
Pruner: pruner,
BlockSize: protocol.BlockSize,
TempNamer: defTempNamer,
TempLifetime: time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
Expand Down Expand Up @@ -1281,10 +1306,10 @@ nextSub:
batch = batch[:0]
}

if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) || pruner.ShouldSkipTruncated(f) {
// File has been ignored or an unsupported symlink. Set invalid bit.
if debug {
l.Debugln("setting invalid bit on ignored", f)
l.Debugln("setting invalid bit on ignored/pruned/unsupported symlink", f)
}
nf := protocol.FileInfo{
Name: f.Name,
Expand Down Expand Up @@ -1633,7 +1658,46 @@ func (m *Model) SetSelections(folder string, selections []string, remove bool) e

cfg.SelectivePatterns = selections
m.cfg.SetFolder(cfg)
return m.cfg.Save()
err := m.cfg.Save()
if err != nil {
return err
}

if !cfg.SelectiveEnabled {
return nil
}

pruner := pruner.New(selections)

m.fmut.Lock()
m.folderPruners[folder] = pruner
m.fmut.Unlock()

// Marks files as invalid.
err = m.ScanFolder(folder)
if !remove || err != nil {
return err
}

m.fmut.RLock()
files := m.folderFiles[folder]
ignores := m.folderIgnores[folder]
m.fmut.RUnlock()

toRemove := make([]string, 0)

files.WithHaveTruncated(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
f := fi.(db.FileInfoTruncated)
if pruner.ShouldSkipTruncated(f) && (ignores == nil || !ignores.Match(f.Name)) {
toRemove = append(toRemove, f.Name)
}
return true
})
for i := len(toRemove) - 1; i >= 0; i-- {
// Best attempt I guess.
osutil.InWritableDir(osutil.Remove, toRemove[i])
}
return nil
}

type jsTreeNode struct {
Expand Down
92 changes: 92 additions & 0 deletions internal/pruner/pruner.go
@@ -0,0 +1,92 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package pruner

import (
"path/filepath"
"strings"

"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/internal/db"
)

type Pruner struct {
dirPatterns []string
filePatterns []string
}

func New(patterns []string) *Pruner {
p := &Pruner{}
for _, pattern := range patterns {
if strings.HasSuffix(pattern, "[^/]+") {
p.filePatterns = append(p.filePatterns, filepath.Dir(pattern))
} else {
p.dirPatterns = append(p.dirPatterns, pattern)
}
}
return p
}

func (p *Pruner) ShouldSkipFile(path string) bool {
if p == nil {
return false
}
for _, pattern := range p.dirPatterns {
// A file which is within a given directory pattern
if strings.HasPrefix(path, pattern) {
return false
}
}

dir := filepath.Dir(path)

for _, pattern := range p.filePatterns {
// A file which matches one of the file patterns
if pattern == dir {
return false
}
}

return true
}

func (p *Pruner) ShouldSkipDirectory(path string) bool {
if p == nil {
return false
}
for _, pattern := range p.dirPatterns {
// A directory which is within a given pattern
if strings.HasPrefix(path, pattern) {
return false
}
// A directory which is required to get down to the given pattern
if strings.HasPrefix(pattern, path) {
return false
}
}
return true
}

func (p *Pruner) ShouldSkip(file protocol.FileInfo) bool {
if p == nil {
return false
}
if file.IsSymlink() || !file.IsDirectory() {
return p.ShouldSkipFile(file.Name)
}
return p.ShouldSkipDirectory(file.Name)
}

func (p *Pruner) ShouldSkipTruncated(file db.FileInfoTruncated) bool {
if p == nil {
return false
}
if file.IsSymlink() || !file.IsDirectory() {
return p.ShouldSkipFile(file.Name)
}
return p.ShouldSkipDirectory(file.Name)
}
13 changes: 12 additions & 1 deletion internal/scanner/walk.go
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/pruner"
"github.com/syncthing/syncthing/internal/symlinks"
"golang.org/x/text/unicode/norm"
)
Expand Down Expand Up @@ -46,6 +47,8 @@ type Walker struct {
BlockSize int
// If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
Matcher *ignore.Matcher
// Pruner used to identify subdirectories which to sync.
Pruner *pruner.Pruner
// If TempNamer is not nil, it is used to ignore tempory files when walking.
TempNamer TempNamer
// Number of hours to keep temporary files for
Expand Down Expand Up @@ -81,7 +84,7 @@ type CurrentFiler interface {
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (chan protocol.FileInfo, error) {
if debug {
l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher)
l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher, w.Pruner)
}

err := checkDir(w.Dir)
Expand Down Expand Up @@ -166,6 +169,14 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
return skip
}

if !info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink {
if w.Pruner.ShouldSkipFile(rn) {
return skip
}
} else if w.Pruner.ShouldSkipDirectory(rn) {
return skip
}

if !utf8.ValidString(rn) {
l.Warnf("File name %q is not in UTF8 encoding; skipping.", rn)
return skip
Expand Down

0 comments on commit 47804eb

Please sign in to comment.