Added support for scanning only one filesystem via files policy (#676)

The new files policy oneFileSystem ignores files that are mounted to
other filesystems similarly to tar's --one-file-system switch. For
example, if this is enabled, backing up / should now automatically
ignore /dev, /proc, etc, so the directory entries themselves don't
appear in the backup. The value of the policy is 'false' by default.

This is implemented by adding a non-windows-field Device (of type
DeviceInfo, reflecting the implementation of Owner) to the Entry
interface. DeviceInfo holds the dev and rdev acquired with stat (same
way as with Owner), but in addition to that it also holds the same
values for the parent directory. It would seem that doing this in some
other way, ie. in ReadDir, would require modifying the ReadDir
interface which seems a too large modification for a feature this
small.

This change introduces a duplication of 'stat' call to the files, as
the Owner feature already does a separate call. I doubt the
performance implications are noticeable, though with some refactoring
both Owner and Device fields could be filled in in one go.

Filling in the field has been placed in fs/localfs/localfs.go where
entryFromChildFileInfo has acquired a third parameter giving the the
parent entry. From that information the Device of the parent is
retrieved, to be passed off to platformSpecificDeviceInfo which does
the rest of the paperwork. Other fs implementations just put in the
default values.

The Dev and Rdev fields returned by the 'stat' call have different
sizes on different platforms, but for convenience they are internally
handled the same. The conversion is done with local_fs_32bit.go and
local_fs_64bit.go which are conditionally compiled on different
platforms.

Finally the actual check of the condition is in ignorefs.go function
shouldIncludeByDevice which is analoguous to the other similarly named
functions.

Co-authored-by: Erkki Seppälä <flux@inside.org>

.gitignore

@@ -22,3 +22,4 @@ node_modules/
 .tmp.*
 .boto
 *.log
+*~

cli/command_policy_edit.go

@@ -40,6 +40,7 @@ const policyEditFilesHelpText = `
   #   "maxFileSize": number
   #   "noParentDotFiles": true
   #   "noParentIgnore": true
+  #   "oneFileSystem": false
 `
 
 const policyEditSchedulingHelpText = `

cli/command_policy_set.go

@@ -60,6 +60,9 @@ var (
 	policySetClearDotIgnore  = policySetCommand.Flag("clear-dot-ignore", "Clear list of paths in the dot-ignore list").Bool()
 	policySetMaxFileSize     = policySetCommand.Flag("max-file-size", "Exclude files above given size").PlaceHolder("N").String()
 
+	// Ignore other mounted fileystems.
+	policyOneFileSystem = policySetCommand.Flag("one-file-system", "Stay in parent filesystem when finding files ('true', 'false', 'inherit')").Enum(booleanEnumValues...)
+
 	// Error handling behavior.
 	policyIgnoreFileErrors      = policySetCommand.Flag("ignore-file-errors", "Ignore errors reading files while traversing ('true', 'false', 'inherit')").Enum(booleanEnumValues...)
 	policyIgnoreDirectoryErrors = policySetCommand.Flag("ignore-dir-errors", "Ignore errors reading directories while traversing ('true', 'false', 'inherit").Enum(booleanEnumValues...)
@@ -189,6 +192,28 @@ func setFilesPolicyFromFlags(ctx context.Context, fp *policy.FilesPolicy, change
 		log(ctx).Infof(" - setting ignore cache dirs to %v\n", val)
 	}
 
+	switch {
+	case *policyOneFileSystem == "":
+	case *policyOneFileSystem == inheritPolicyString:
+		*changeCount++
+
+		fp.OneFileSystem = nil
+
+		printStderr(" - inherit one file system from parent\n")
+
+	default:
+		val, err := strconv.ParseBool(*policyOneFileSystem)
+		if err != nil {
+			return err
+		}
+
+		*changeCount++
+
+		fp.OneFileSystem = &val
+
+		printStderr(" - setting one file system to %v\n", val)
+	}
+
 	return nil
 }
 

cli/command_policy_show.go

@@ -160,6 +160,12 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) {
 				return pol.FilesPolicy.MaxFileSize != 0
 			}))
 	}
+
+	printStdout("  Scan one filesystem only:       %5v       %v\n",
+		p.FilesPolicy.OneFileSystemOrDefault(false),
+		getDefinitionPoint(parents, func(pol *policy.Policy) bool {
+			return pol.FilesPolicy.OneFileSystem != nil
+		}))
 }
 
 func printErrorHandlingPolicy(p *policy.Policy, parents []*policy.Policy) {

fs/entry.go

@@ -13,6 +13,7 @@ import (
 type Entry interface {
 	os.FileInfo
 	Owner() OwnerInfo
+	Device() DeviceInfo
 }
 
 // OwnerInfo describes owner of a filesystem entry.
@@ -21,6 +22,12 @@ type OwnerInfo struct {
 	GroupID uint32
 }
 
+// DeviceInfo describes the device this filesystem entry is on.
+type DeviceInfo struct {
+	Dev  uint64
+	Rdev uint64
+}
+
 // Entries is a list of entries sorted by name.
 type Entries []Entry
 

fs/ignorefs/ignorefs.go

@@ -28,6 +28,8 @@ type ignoreContext struct {
 	dotIgnoreFiles []string         // which files to look for more ignore rules
 	matchers       []ignore.Matcher // current set of rules to ignore files
 	maxFileSize    int64            // maximum size of file allowed
+
+	oneFileSystem bool // should we enter other mounted filesystems
 }
 
 func (c *ignoreContext) shouldIncludeByName(path string, e fs.Entry) bool {
@@ -48,6 +50,14 @@ func (c *ignoreContext) shouldIncludeByName(path string, e fs.Entry) bool {
 	return c.parent.shouldIncludeByName(path, e)
 }
 
+func (c *ignoreContext) shouldIncludeByDevice(e fs.Entry, parent *ignoreDirectory) bool {
+	if !c.oneFileSystem {
+		return true
+	}
+
+	return e.Device().Dev == parent.Device().Dev
+}
+
 type ignoreDirectory struct {
 	relativePath  string
 	parentContext *ignoreContext
@@ -132,6 +142,10 @@ func (d *ignoreDirectory) Readdir(ctx context.Context) (fs.Entries, error) {
 			continue
 		}
 
+		if !thisContext.shouldIncludeByDevice(e, d) {
+			continue
+		}
+
 		if dir, ok := e.(fs.Directory); ok {
 			e = &ignoreDirectory{d.relativePath + "/" + e.Name(), thisContext, d.policyTree.Child(e.Name()), dir}
 		}
@@ -168,6 +182,7 @@ func (d *ignoreDirectory) buildContext(ctx context.Context, entries fs.Entries)
 		onIgnore:       d.parentContext.onIgnore,
 		dotIgnoreFiles: effectiveDotIgnoreFiles,
 		maxFileSize:    d.parentContext.maxFileSize,
+		oneFileSystem:  d.parentContext.oneFileSystem,
 	}
 
 	if pol != nil {
@@ -197,6 +212,8 @@ func (c *ignoreContext) overrideFromPolicy(fp *policy.FilesPolicy, dirPath strin
 		c.maxFileSize = fp.MaxFileSize
 	}
 
+	c.oneFileSystem = fp.OneFileSystemOrDefault(false)
+
 	// append policy-level rules
 	for _, rule := range fp.IgnoreRules {
 		m, err := ignore.ParseGitIgnore(dirPath, rule)

fs/ignorefs/ignorefs_test.go

@@ -28,15 +28,18 @@ func setupFilesystem() *mockfs.Directory {
 	root.AddFile("ignored-by-rule", dummyFileContents, 0)
 	root.AddFile("largefile1", tooLargeFileContents, 0)
 
+	dev1 := fs.DeviceInfo{Dev: 1, Rdev: 0}
+	dev2 := fs.DeviceInfo{Dev: 2, Rdev: 0}
+
 	d1 := root.AddDir("bin", 0)
-	d2 := root.AddDir("pkg", 0)
-	d3 := root.AddDir("src", 0)
+	d2 := root.AddDirDevice("pkg", 0, dev1)
+	d3 := root.AddDirDevice("src", 0, dev2)
 
 	d1.AddFile("some-bin", dummyFileContents, 0)
-	d2.AddFile("some-pkg", dummyFileContents, 0)
+	d2.AddFileDevice("some-pkg", dummyFileContents, 0, dev1)
 
-	d4 := d3.AddDir("some-src", 0)
-	d4.AddFile("f1", dummyFileContents, 0)
+	d4 := d3.AddDirDevice("some-src", 0, dev2)
+	d4.AddFileDevice("f1", dummyFileContents, 0, dev2)
 
 	return root
 }
@@ -79,6 +82,16 @@ var rootAndSrcPolicy = policy.BuildTree(map[string]*policy.Policy{
 	},
 }, policy.DefaultPolicy)
 
+var trueValue = true
+
+var oneFileSystemPolicy = policy.BuildTree(map[string]*policy.Policy{
+	".": {
+		FilesPolicy: policy.FilesPolicy{
+			OneFileSystem: &trueValue,
+		},
+	},
+}, policy.DefaultPolicy)
+
 var cases = []struct {
 	desc         string
 	policyTree   *policy.Tree
@@ -216,6 +229,18 @@ var cases = []struct {
 			"./src/some-src/f1",
 		},
 	},
+	{
+		desc:       "policy with one-file-system",
+		policyTree: oneFileSystemPolicy,
+		addedFiles: nil,
+		ignoredFiles: []string{
+			"./pkg/",
+			"./pkg/some-pkg",
+			"./src/",
+			"./src/some-src/",
+			"./src/some-src/f1",
+		},
+	},
 }
 
 func TestIgnoreFS(t *testing.T) {

fs/localfs/local_fs.go

@@ -36,6 +36,7 @@ type filesystemEntry struct {
 	mtimeNanos int64
 	mode       os.FileMode
 	owner      fs.OwnerInfo
+	device     fs.DeviceInfo
 
 	parentDir string
 }
@@ -72,6 +73,10 @@ func (e *filesystemEntry) Owner() fs.OwnerInfo {
 	return e.owner
 }
 
+func (e *filesystemEntry) Device() fs.DeviceInfo {
+	return e.device
+}
+
 var _ os.FileInfo = (*filesystemEntry)(nil)
 
 func newEntry(fi os.FileInfo, parentDir string) filesystemEntry {
@@ -81,6 +86,7 @@ func newEntry(fi os.FileInfo, parentDir string) filesystemEntry {
 		fi.ModTime().UnixNano(),
 		fi.Mode(),
 		platformSpecificOwnerInfo(fi),
+		platformSpecificDeviceInfo(fi),
 		parentDir,
 	}
 }

fs/localfs/local_fs_32bit.go

@@ -0,0 +1,8 @@
+// +build !windows
+// +build !amd64,!arm64,!arm darwin
+
+package localfs
+
+func platformSpecificWidenDev(dev int32) uint64 {
+	return uint64(dev)
+}

fs/localfs/local_fs_64bit.go

@@ -0,0 +1,9 @@
+// +build !windows
+// +build !darwin
+// +build amd64 arm64 arm
+
+package localfs
+
+func platformSpecificWidenDev(dev uint64) uint64 {
+	return dev
+}

fs/localfs/local_fs_nonwindows.go

@@ -18,3 +18,14 @@ func platformSpecificOwnerInfo(fi os.FileInfo) fs.OwnerInfo {
 
 	return oi
 }
+
+func platformSpecificDeviceInfo(fi os.FileInfo) fs.DeviceInfo {
+	var oi fs.DeviceInfo
+	if stat, ok := fi.Sys().(*syscall.Stat_t); ok {
+		// not making a separate type for 32-bit platforms here..
+		oi.Dev = platformSpecificWidenDev(stat.Dev)
+		oi.Rdev = platformSpecificWidenDev(stat.Rdev)
+	}
+
+	return oi
+}

fs/localfs/local_fs_windows.go

@@ -9,3 +9,7 @@ import (
 func platformSpecificOwnerInfo(fi os.FileInfo) fs.OwnerInfo {
 	return fs.OwnerInfo{}
 }
+
+func platformSpecificDeviceInfo(fi os.FileInfo) fs.DeviceInfo {
+	return fs.DeviceInfo{}
+}

htmlui/src/PolicyEditor.js

@@ -182,6 +182,9 @@ export class PolicyEditor extends Component {
                         <Form.Row>
                             {OptionalBoolean(this, "Ignore Well-Known Cache Directories", "policy.files.ignoreCacheDirs", "inherit from parent")}
                         </Form.Row>
+                        <Form.Row>
+                            {OptionalBoolean(this, "Scan only one filesystem", "policy.files.oneFileSystem", "inherit from parent")}
+                        </Form.Row>
                     </div>
                 </Tab>
                 <Tab eventKey="errors" title="Errors">

internal/diff/diff.go

@@ -191,6 +191,8 @@ func compareEntry(e1, e2 fs.Entry, fullpath string, out io.Writer) bool {
 		fmt.Fprintln(out, fullpath, "owner groups differ: ", o1.GroupID, o2.GroupID)
 	}
 
+	// don't compare filesystem boundaries (e1.Device()), it's pretty useless and is not stored in backups
+
 	return equal
 }
 

internal/mockfs/mockfs.go

@@ -34,36 +34,41 @@ type entry struct {
 	size    int64
 	modTime time.Time
 	owner   fs.OwnerInfo
+	device  fs.DeviceInfo
 }
 
-func (e entry) Name() string {
+func (e *entry) Name() string {
 	return e.name
 }
 
-func (e entry) IsDir() bool {
+func (e *entry) IsDir() bool {
 	return e.mode.IsDir()
 }
 
-func (e entry) Mode() os.FileMode {
+func (e *entry) Mode() os.FileMode {
 	return e.mode
 }
 
-func (e entry) ModTime() time.Time {
+func (e *entry) ModTime() time.Time {
 	return e.modTime
 }
 
-func (e entry) Size() int64 {
+func (e *entry) Size() int64 {
 	return e.size
 }
 
-func (e entry) Sys() interface{} {
+func (e *entry) Sys() interface{} {
 	return nil
 }
 
-func (e entry) Owner() fs.OwnerInfo {
+func (e *entry) Owner() fs.OwnerInfo {
 	return e.owner
 }
 
+func (e *entry) Device() fs.DeviceInfo {
+	return e.device
+}
+
 // Directory is mock in-memory implementation of fs.Directory.
 type Directory struct {
 	entry
@@ -97,6 +102,26 @@ func (imd *Directory) AddFile(name string, content []byte, permissions os.FileMo
 	return file
 }
 
+// AddFileDevice adds a mock file with the specified name, content, permissions, and device info.
+func (imd *Directory) AddFileDevice(name string, content []byte, permissions os.FileMode, deviceInfo fs.DeviceInfo) *File {
+	imd, name = imd.resolveSubdir(name)
+	file := &File{
+		entry: entry{
+			name:   name,
+			mode:   permissions,
+			size:   int64(len(content)),
+			device: deviceInfo,
+		},
+		source: func() (ReaderSeekerCloser, error) {
+			return readerSeekerCloser{bytes.NewReader(content)}, nil
+		},
+	}
+
+	imd.addChild(file)
+
+	return file
+}
+
 // AddDir adds a fake directory with a given name and permissions.
 func (imd *Directory) AddDir(name string, permissions os.FileMode) *Directory {
 	imd, name = imd.resolveSubdir(name)
@@ -113,6 +138,23 @@ func (imd *Directory) AddDir(name string, permissions os.FileMode) *Directory {
 	return subdir
 }
 
+// AddDirDevice adds a fake directory with a given name and permissions.
+func (imd *Directory) AddDirDevice(name string, permissions os.FileMode, deviceInfo fs.DeviceInfo) *Directory {
+	imd, name = imd.resolveSubdir(name)
+
+	subdir := &Directory{
+		entry: entry{
+			name:   name,
+			mode:   permissions | os.ModeDir,
+			device: deviceInfo,
+		},
+	}
+
+	imd.addChild(subdir)
+
+	return subdir
+}
+
 func (imd *Directory) addChild(e fs.Entry) {
 	if strings.Contains(e.Name(), "/") {
 		panic("child name cannot contain '/'")