implemented Cache Directory Tagging Specification + CLI + UI (#565)

Fixes #564

cli: added 'kopia policy set --ignore-cache-dirs' option to control
whether to ignore caches (global default=true)

ui: added checkbox to control 'Ignore Cache Dirs' in policy editor

ignorefs: moved ignoring cache directories to ignorefs layer

Co-authored-by: Julio López <julio+gh@kasten.io>

cli/command_policy_set.go

@@ -37,6 +37,8 @@ var (
 	policySetRemoveIgnore = policySetCommand.Flag("remove-ignore", "List of paths to remove from the ignore list").PlaceHolder("PATTERN").Strings()
 	policySetClearIgnore  = policySetCommand.Flag("clear-ignore", "Clear list of paths in the ignore list").Bool()
 
+	policyIgnoreCacheDirs = policySetCommand.Flag("ignore-cache-dirs", "Ignore cache directories ('true', 'false', 'inherit')").Enum(booleanEnumValues...)
+
 	// Name of compression algorithm.
 	policySetCompressionAlgorithm = policySetCommand.Flag("compression", "Compression algorithm").Enum(supportedCompressionAlgorithms()...)
 	policySetCompressionMinSize   = policySetCommand.Flag("compression-min-size", "Min size of file to attempt compression for").String()
@@ -114,7 +116,9 @@ func setPolicyFromFlags(p *policy.Policy, changeCount *int) error {
 		return errors.Wrap(err, "retention policy")
 	}
 
-	setFilesPolicyFromFlags(&p.FilesPolicy, changeCount)
+	if err := setFilesPolicyFromFlags(&p.FilesPolicy, changeCount); err != nil {
+		return errors.Wrap(err, "files policy")
+	}
 
 	if err := setErrorHandlingPolicyFromFlags(&p.ErrorHandlingPolicy, changeCount); err != nil {
 		return errors.Wrap(err, "error handling policy")
@@ -142,7 +146,7 @@ func setPolicyFromFlags(p *policy.Policy, changeCount *int) error {
 	return nil
 }
 
-func setFilesPolicyFromFlags(fp *policy.FilesPolicy, changeCount *int) {
+func setFilesPolicyFromFlags(fp *policy.FilesPolicy, changeCount *int) error {
 	if *policySetClearDotIgnore {
 		*changeCount++
 
@@ -162,6 +166,30 @@ func setFilesPolicyFromFlags(fp *policy.FilesPolicy, changeCount *int) {
 	} else {
 		fp.IgnoreRules = addRemoveDedupeAndSort("ignored files", fp.IgnoreRules, *policySetAddIgnore, *policySetRemoveIgnore, changeCount)
 	}
+
+	switch {
+	case *policyIgnoreCacheDirs == "":
+	case *policyIgnoreCacheDirs == inheritPolicyString:
+		*changeCount++
+
+		fp.IgnoreCacheDirs = nil
+
+		printStderr(" - inherit ignoring cache dirs from parent\n")
+
+	default:
+		val, err := strconv.ParseBool(*policyIgnoreCacheDirs)
+		if err != nil {
+			return err
+		}
+
+		*changeCount++
+
+		fp.IgnoreCacheDirs = &val
+
+		printStderr(" - setting ignore cache dirs to %v\n", val)
+	}
+
+	return nil
 }
 
 func setErrorHandlingPolicyFromFlags(fp *policy.ErrorHandlingPolicy, changeCount *int) error {

cli/command_policy_show.go

@@ -123,6 +123,12 @@ func printRetentionPolicy(p *policy.Policy, parents []*policy.Policy) {
 func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) {
 	printStdout("Files policy:\n")
 
+	printStdout("  Ignore cache directories:       %5v       %v\n",
+		p.FilesPolicy.IgnoreCacheDirectoriesOrDefault(true),
+		getDefinitionPoint(parents, func(pol *policy.Policy) bool {
+			return pol.FilesPolicy.IgnoreCacheDirs != nil
+		}))
+
 	if len(p.FilesPolicy.IgnoreRules) > 0 {
 		printStdout("  Ignore rules:\n")
 	} else {

cli/command_snapshot_estimate.go

@@ -82,12 +82,14 @@ func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error
 	eb := makeBuckets()
 
 	onIgnoredFile := func(relativePath string, e fs.Entry) {
-		log(ctx).Infof("ignoring %v", relativePath)
 		eb.add(relativePath, e.Size())
 
 		if e.IsDir() {
 			stats.ExcludedDirCount++
+
+			log(ctx).Infof("excluded dir %v", relativePath)
 		} else {
+			log(ctx).Infof("excluded file %v (%v)", relativePath, units.BytesStringBase10(e.Size()))
 			stats.ExcludedFileCount++
 			stats.ExcludedTotalFileSize += e.Size()
 		}

fs/ignorefs/ignorefs.go

@@ -10,9 +10,13 @@ import (
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/internal/ignore"
+	"github.com/kopia/kopia/repo"
+	"github.com/kopia/kopia/repo/logging"
 	"github.com/kopia/kopia/snapshot/policy"
 )
 
+var log = logging.GetContextLoggerFunc("ignorefs")
+
 // IgnoreCallback is a function called by ignorefs to report whenever a file or directory is being ignored while listing its parent.
 type IgnoreCallback func(path string, metadata fs.Entry)
 
@@ -52,12 +56,66 @@ type ignoreDirectory struct {
 	fs.Directory
 }
 
+func isCorrectCacheDirSignature(ctx context.Context, f fs.File) (bool, error) {
+	const (
+		validSignature    = repo.CacheDirMarkerHeader
+		validSignatureLen = len(validSignature)
+	)
+
+	if f.Size() < int64(validSignatureLen) {
+		return false, nil
+	}
+
+	r, err := f.Open(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	defer r.Close() //nolint:errcheck
+
+	sig := make([]byte, validSignatureLen)
+
+	if _, err := r.Read(sig); err != nil {
+		return false, err
+	}
+
+	return string(sig) == validSignature, nil
+}
+
+func (d *ignoreDirectory) skipCacheDirectory(ctx context.Context, entries fs.Entries, relativePath string, policyTree *policy.Tree) fs.Entries {
+	if !policyTree.EffectivePolicy().FilesPolicy.IgnoreCacheDirectoriesOrDefault(true) {
+		return entries
+	}
+
+	f, ok := entries.FindByName(repo.CacheDirMarkerFile).(fs.File)
+	if ok {
+		correct, err := isCorrectCacheDirSignature(ctx, f)
+		if err != nil {
+			log(ctx).Debugf("unable to check cache dir signature, assuming not a cache directory: %v", err)
+			return entries
+		}
+
+		if correct {
+			// if the given directory contains a marker file used for kopia cache, pretend the directory was empty.
+			for _, oi := range d.parentContext.onIgnore {
+				oi(relativePath, d)
+			}
+
+			return nil
+		}
+	}
+
+	return entries
+}
+
 func (d *ignoreDirectory) Readdir(ctx context.Context) (fs.Entries, error) {
 	entries, err := d.Directory.Readdir(ctx)
 	if err != nil {
 		return nil, err
 	}
 
+	entries = d.skipCacheDirectory(ctx, entries, d.relativePath, d.policyTree)
+
 	thisContext, err := d.buildContext(ctx, entries)
 	if err != nil {
 		return nil, err
@@ -125,6 +183,7 @@ func (d *ignoreDirectory) buildContext(ctx context.Context, entries fs.Entries)
 	return newic, nil
 }
 
+// nolint:gocritic
 func (c *ignoreContext) overrideFromPolicy(fp policy.FilesPolicy, dirPath string) error {
 	if fp.NoParentDotIgnoreFiles {
 		c.dotIgnoreFiles = nil

htmlui/src/PolicyEditor.js

@@ -179,6 +179,9 @@ export class PolicyEditor extends Component {
                             {RequiredBoolean(this, "Ignore Parent Rules", "policy.files.noParentIgnore")}
                             {RequiredBoolean(this, "Ignore Parent Rule Files", "policy.files.noParentDotFiles")}
                         </Form.Row>
+                        <Form.Row>
+                            {OptionalBoolean(this, "Ignore Well-Known Cache Directories", "policy.files.ignoreCacheDirs", "inherit from parent")}
+                        </Form.Row>
                     </div>
                 </Tab>
                 <Tab eventKey="errors" title="Errors">

repo/open.go

@@ -22,7 +22,22 @@ import (
 )
 
 // CacheDirMarkerFile is the name of the marker file indicating a directory contains Kopia caches.
-const CacheDirMarkerFile = ".kopia-cache"
+// See https://bford.info/cachedir/
+const CacheDirMarkerFile = "CACHEDIR.TAG"
+
+// CacheDirMarkerHeader is the header signature for cache dir marker files.
+const CacheDirMarkerHeader = "Signature: 8a477f597d28d172789f06886806bc55"
+
+const cacheDirMarkerContents = CacheDirMarkerHeader + `
+#
+# This file is a cache directory tag created by Kopia - Fast And Secure Open-Source Backup.
+#
+# For information about Kopia, see:
+#   https://kopia.io
+#
+# For information about cache directory tags, see:
+#   http://www.brynosaurus.com/cachedir/
+`
 
 var log = logging.GetContextLoggerFunc("kopia/repo")
 
@@ -193,7 +208,14 @@ func writeCacheMarker(cacheDir string) error {
 	}
 
 	markerFile := filepath.Join(cacheDir, CacheDirMarkerFile)
-	if _, err := os.Stat(markerFile); !os.IsNotExist(err) {
+
+	st, err := os.Stat(markerFile)
+	if err == nil && st.Size() >= int64(len(cacheDirMarkerContents)) {
+		// ok
+		return nil
+	}
+
+	if !os.IsNotExist(err) {
 		return err
 	}
 
@@ -202,6 +224,10 @@ func writeCacheMarker(cacheDir string) error {
 		return err
 	}
 
+	if _, err := f.WriteString(cacheDirMarkerContents); err != nil {
+		return errors.Wrap(err, "unable to write cachedir marker contents")
+	}
+
 	return f.Close()
 }
 

snapshot/policy/files_policy.go

@@ -8,10 +8,13 @@ type FilesPolicy struct {
 	DotIgnoreFiles         []string `json:"ignoreDotFiles,omitempty"`
 	NoParentDotIgnoreFiles bool     `json:"noParentDotFiles,omitempty"`
 
+	IgnoreCacheDirs *bool `json:"ignoreCacheDirs,omitempty"`
+
 	MaxFileSize int64 `json:"maxFileSize,omitempty"`
 }
 
 // Merge applies default values from the provided policy.
+// nolint:gocritic
 func (p *FilesPolicy) Merge(src FilesPolicy) {
 	if p.MaxFileSize == 0 {
 		p.MaxFileSize = src.MaxFileSize
@@ -24,6 +27,19 @@ func (p *FilesPolicy) Merge(src FilesPolicy) {
 	if len(p.DotIgnoreFiles) == 0 {
 		p.DotIgnoreFiles = src.DotIgnoreFiles
 	}
+
+	if p.IgnoreCacheDirs == nil {
+		p.IgnoreCacheDirs = src.IgnoreCacheDirs
+	}
+}
+
+// IgnoreCacheDirectoriesOrDefault gets the value of IgnoreCacheDirs or the provided default if not set.
+func (p *FilesPolicy) IgnoreCacheDirectoriesOrDefault(def bool) bool {
+	if p.IgnoreCacheDirs == nil {
+		return def
+	}
+
+	return *p.IgnoreCacheDirs
 }
 
 // defaultFilesPolicy is the default file ignore policy.

snapshot/snapshotfs/upload.go

@@ -688,7 +688,7 @@ func maybeReadDirectoryEntries(ctx context.Context, dir fs.Directory) fs.Entries
 		return nil
 	}
 
-	return skipCacheDirectory(ent)
+	return ent
 }
 
 func uniqueDirectories(dirs []fs.Directory) []fs.Directory {
@@ -746,8 +746,6 @@ func uploadDirInternal(
 		return "", fs.DirectorySummary{}, dirReadError{direrr}
 	}
 
-	entries = skipCacheDirectory(entries)
-
 	var prevEntries []fs.Entries
 
 	for _, d := range uniqueDirectories(previousDirs) {
@@ -784,15 +782,6 @@ func uploadDirInternal(
 	return oid, *dirManifest.Summary, err
 }
 
-func skipCacheDirectory(entries fs.Entries) fs.Entries {
-	if entries.FindByName(repo.CacheDirMarkerFile) != nil {
-		// if the given directory contains a marker file used for kopia cache, pretend the directory was empty.
-		return nil
-	}
-
-	return entries
-}
-
 func (u *Uploader) maybeIgnoreFileReadError(err error, output chan dirEntryOrError, entryRelativePath string, policyTree *policy.Tree) error {
 	errHandlingPolicy := policyTree.EffectivePolicy().ErrorHandlingPolicy
 

tests/testenv/faketimeserver.go

@@ -52,7 +52,7 @@ func (s *FakeTimeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 func NewFakeTimeServer(startTime time.Time, step time.Duration) *FakeTimeServer {
 	return &FakeTimeServer{
 		nextTimeChunk:   startTime,
-		timeChunkLength: 100 * step, // nolint:mnd
+		timeChunkLength: 100 * step, // nolint:gomnd
 		step:            step,
 	}
 }