Skip to content

Commit

Permalink
Add pruner
Browse files Browse the repository at this point in the history
  • Loading branch information
AudriusButkevicius committed Apr 27, 2015
1 parent 7a02ade commit 80cd780
Show file tree
Hide file tree
Showing 4 changed files with 136 additions and 14 deletions.
22 changes: 16 additions & 6 deletions internal/auto/gui.files.go

Large diffs are not rendered by default.

30 changes: 23 additions & 7 deletions internal/model/model.go
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/syncthing/syncthing/internal/events"
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/pruner"
"github.com/syncthing/syncthing/internal/scanner"
"github.com/syncthing/syncthing/internal/stats"
"github.com/syncthing/syncthing/internal/symlinks"
Expand Down Expand Up @@ -72,6 +73,7 @@ type Model struct {
deviceFolders map[protocol.DeviceID][]string // deviceID -> folders
deviceStatRefs map[protocol.DeviceID]*stats.DeviceStatisticsReference // deviceID -> statsRef
folderIgnores map[string]*ignore.Matcher // folder -> matcher object
folderPruners map[string]*pruner.Pruner // folder -> pruner object
folderRunners map[string]service // folder -> puller or scanner
folderStatRefs map[string]*stats.FolderStatisticsReference // folder -> statsRef
fmut sync.RWMutex // protects the above
Expand Down Expand Up @@ -109,6 +111,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
deviceFolders: make(map[protocol.DeviceID][]string),
deviceStatRefs: make(map[protocol.DeviceID]*stats.DeviceStatisticsReference),
folderIgnores: make(map[string]*ignore.Matcher),
folderPruners: make(map[string]*pruner.Pruner),
folderRunners: make(map[string]service),
folderStatRefs: make(map[string]*stats.FolderStatisticsReference),
protoConn: make(map[protocol.DeviceID]protocol.Connection),
Expand Down Expand Up @@ -926,7 +929,7 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
m.fmut.RLock()
for _, folder := range m.deviceFolders[deviceID] {
fs := m.folderFiles[folder]
go sendIndexes(protoConn, folder, fs, m.folderIgnores[folder])
go sendIndexes(protoConn, folder, fs, m.folderIgnores[folder], m.folderPruners[folder])
}
m.fmut.RUnlock()
m.pmut.Unlock()
Expand Down Expand Up @@ -967,7 +970,7 @@ func (m *Model) receivedFile(folder, filename string) {
m.folderStatRef(folder).ReceivedFile(filename)
}

func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) {
func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher, pruner *pruner.Pruner) {
deviceID := conn.ID()
name := conn.Name()
var err error
Expand All @@ -976,23 +979,23 @@ func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignore
l.Debugf("sendIndexes for %s-%s/%q starting", deviceID, name, folder)
}

minLocalVer, err := sendIndexTo(true, 0, conn, folder, fs, ignores)
minLocalVer, err := sendIndexTo(true, 0, conn, folder, fs, ignores, pruner)

for err == nil {
time.Sleep(5 * time.Second)
if fs.LocalVersion(protocol.LocalDeviceID) <= minLocalVer {
continue
}

minLocalVer, err = sendIndexTo(false, minLocalVer, conn, folder, fs, ignores)
minLocalVer, err = sendIndexTo(false, minLocalVer, conn, folder, fs, ignores, pruner)
}

if debug {
l.Debugf("sendIndexes for %s-%s/%q exiting: %v", deviceID, name, folder, err)
}
}

func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) (int64, error) {
func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher, pruner *pruner.Pruner) (int64, error) {
deviceID := conn.ID()
name := conn.Name()
batch := make([]protocol.FileInfo, 0, indexBatchSize)
Expand All @@ -1010,6 +1013,13 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
maxLocalVer = f.LocalVersion
}

if pruner != nil && pruner.ShouldSkip(f) {
if debug {
l.Debugln("not sending update for pruned file", f)
}
return true
}

if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
if debug {
l.Debugln("not sending update for ignored/unsupported symlink", f)
Expand Down Expand Up @@ -1108,6 +1118,10 @@ func (m *Model) AddFolder(cfg config.FolderConfiguration) {
_ = ignores.Load(filepath.Join(cfg.Path(), ".stignore")) // Ignore error, there might not be an .stignore
m.folderIgnores[cfg.ID] = ignores

if cfg.SelectiveEnabled {
m.folderPruners[cfg.ID] = pruner.New(cfg.SelectivePatterns)
}

m.addedFolder = true
m.fmut.Unlock()
}
Expand Down Expand Up @@ -1167,6 +1181,7 @@ func (m *Model) ScanFolderSubs(folder string, subs []string) error {
fs := m.folderFiles[folder]
folderCfg := m.folderCfgs[folder]
ignores := m.folderIgnores[folder]
pruner := m.folderPruners[folder]
runner, ok := m.folderRunners[folder]
m.fmut.Unlock()

Expand Down Expand Up @@ -1206,6 +1221,7 @@ nextSub:
Dir: folderCfg.Path(),
Subs: subs,
Matcher: ignores,
Pruner: pruner,
BlockSize: protocol.BlockSize,
TempNamer: defTempNamer,
TempLifetime: time.Duration(m.cfg.Options().KeepTemporariesH) * time.Hour,
Expand Down Expand Up @@ -1281,10 +1297,10 @@ nextSub:
batch = batch[:0]
}

if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) || (pruner != nil && pruner.ShouldSkipTruncated(f)) {
// File has been ignored or an unsupported symlink. Set invalid bit.
if debug {
l.Debugln("setting invalid bit on ignored", f)
l.Debugln("setting invalid bit on ignored/pruned/unsupported symlink", f)
}
nf := protocol.FileInfo{
Name: f.Name,
Expand Down
83 changes: 83 additions & 0 deletions internal/pruner/pruner.go
@@ -0,0 +1,83 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package pruner

import (
"path/filepath"
"strings"

"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/internal/db"
)

type Pruner struct {
dirPatterns []string
filePatterns []string
}

func New(patterns []string) *Pruner {
p := &Pruner{
dirPatterns: make([]string, 0),
filePatterns: make([]string, 0),
}
for _, pattern := range patterns {
if strings.HasSuffix(pattern, "[^/]+") {
p.filePatterns = append(p.filePatterns, filepath.Dir(pattern))
} else {
p.dirPatterns = append(p.dirPatterns, pattern)
}
}
return p
}

func (p *Pruner) ShouldSkipFile(path string) bool {
for _, pattern := range p.dirPatterns {
// A file which is within a given directory pattern
if strings.HasPrefix(path, pattern) {
return false
}
}

dir := filepath.Dir(path)

for _, pattern := range p.filePatterns {
// A file which matches one of the file patterns
if pattern == dir {
return false
}
}

return true
}

func (p *Pruner) ShouldSkipDirectory(path string) bool {
for _, pattern := range p.dirPatterns {
// A directory which is within a given pattern
if strings.HasPrefix(path, pattern) {
return false
}
// A directory which is required to get down to the given pattern
if strings.HasPrefix(pattern, path) {
return false
}
}
return true
}

func (p *Pruner) ShouldSkip(file protocol.FileInfo) bool {
if file.IsSymlink() || !file.IsDirectory() {
return p.ShouldSkipFile(file.Name)
}
return p.ShouldSkipDirectory(file.Name)
}

func (p *Pruner) ShouldSkipTruncated(file db.FileInfoTruncated) bool {
if file.IsSymlink() || !file.IsDirectory() {
return p.ShouldSkipFile(file.Name)
}
return p.ShouldSkipDirectory(file.Name)
}
15 changes: 14 additions & 1 deletion internal/scanner/walk.go
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/syncthing/protocol"
"github.com/syncthing/syncthing/internal/ignore"
"github.com/syncthing/syncthing/internal/osutil"
"github.com/syncthing/syncthing/internal/pruner"
"github.com/syncthing/syncthing/internal/symlinks"
"golang.org/x/text/unicode/norm"
)
Expand Down Expand Up @@ -46,6 +47,8 @@ type Walker struct {
BlockSize int
// If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
Matcher *ignore.Matcher
// If Pruner is not nil, it is used to identify subdirectories which to sync.
Pruner *pruner.Pruner
// If TempNamer is not nil, it is used to ignore tempory files when walking.
TempNamer TempNamer
// Number of hours to keep temporary files for
Expand Down Expand Up @@ -81,7 +84,7 @@ type CurrentFiler interface {
// file system. Files are blockwise hashed.
func (w *Walker) Walk() (chan protocol.FileInfo, error) {
if debug {
l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher)
l.Debugln("Walk", w.Dir, w.Subs, w.BlockSize, w.Matcher, w.Pruner)
}

err := checkDir(w.Dir)
Expand Down Expand Up @@ -166,6 +169,16 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
return skip
}

if w.Pruner != nil {
if !info.IsDir() || info.Mode()&os.ModeSymlink == os.ModeSymlink {
if w.Pruner.ShouldSkipFile(rn) {
return skip
}
} else if w.Pruner.ShouldSkipDirectory(rn) {
return skip
}
}

if !utf8.ValidString(rn) {
l.Warnf("File name %q is not in UTF8 encoding; skipping.", rn)
return skip
Expand Down

0 comments on commit 80cd780

Please sign in to comment.