Folders: Ignore directories listed in a .ppignore file #4237

Signed-off-by: Michael Mayer <michael@photoprism.app>

internal/photoprism/convert.go

@@ -70,7 +70,7 @@ func (c *Convert) Start(dir string, ext []string, force bool) (err error) {
 	done := make(fs.Done)
 	ignore := fs.NewIgnoreList(fs.PPIgnoreFilename, true, false)
 
-	if err = ignore.Dir(dir); err != nil {
+	if err = ignore.Path(dir); err != nil {
 		log.Infof("convert: %s", err)
 	}
 

internal/photoprism/import.go

@@ -112,7 +112,7 @@ func (imp *Import) Start(opt ImportOptions) fs.Done {
 	skipRaw := imp.conf.DisableRaw()
 	ignore := fs.NewIgnoreList(fs.PPIgnoreFilename, true, false)
 
-	if err := ignore.Dir(importPath); err != nil {
+	if err := ignore.Path(importPath); err != nil {
 		log.Infof("import: %s", err)
 	}
 

internal/photoprism/index.go

@@ -135,7 +135,7 @@ func (ind *Index) Start(o IndexOptions) (found fs.Done, updated int) {
 	skipRaw := ind.conf.DisableRaw()
 	ignore := fs.NewIgnoreList(fs.PPIgnoreFilename, true, false)
 
-	if err := ignore.Dir(originalsPath); err != nil {
+	if err := ignore.Path(originalsPath); err != nil {
 		log.Infof("index: %s", err)
 	}
 

internal/photoprism/thumbs.go

@@ -134,7 +134,7 @@ func (w *Thumbs) Dir(dir string, force bool) (fs.Done, error) {
 
 	log.Infof("thumbs: processing %s", clean.Log(dir))
 
-	if err := ignore.Dir(dir); err != nil {
+	if err := ignore.Path(dir); err != nil {
 		log.Infof("thumbs: %s", err)
 	}
 

pkg/fastwalk/fastwalk.go

@@ -48,7 +48,7 @@ var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
 //   - fastWalk can follow symlinks if walkFn returns the TraverseLink
 //     sentinel error. It is the walkFn's responsibility to prevent
 //     fastWalk from going into symlink cycles.
-func Walk(root string, walkFn func(path string, typ os.FileMode) error) error {
+func Walk(root string, walkFn func(path string, mode os.FileMode) error) error {
 	// TODO(bradfitz): make numWorkers configurable? We used a
 	// minimum of 4 to give the kernel more info about multiple
 	// things we want, in hopes its I/O scheduling can take
@@ -138,7 +138,7 @@ func (w *walker) doWork(wg *sync.WaitGroup) {
 }
 
 type walker struct {
-	fn func(path string, typ os.FileMode) error
+	fn func(path string, mode os.FileMode) error
 
 	donec    chan struct{} // closed on fastWalk's return
 	workc    chan walkItem // to workers
@@ -158,22 +158,22 @@ func (w *walker) enqueue(it walkItem) {
 	}
 }
 
-func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
+func (w *walker) onDirEnt(dirName, baseName string, mode os.FileMode) error {
 	joined := dirName + string(os.PathSeparator) + baseName
-	if typ == os.ModeDir {
+	if mode == os.ModeDir {
 		w.enqueue(walkItem{dir: joined})
 		return nil
 	}
 
-	err := w.fn(joined, typ)
-	if typ == os.ModeSymlink {
-		if err == ErrTraverseLink {
+	err := w.fn(joined, mode)
+	if mode == os.ModeSymlink {
+		if errors.Is(err, ErrTraverseLink) {
 			// Set callbackDone so we don't call it twice for both the
 			// symlink-as-symlink and the symlink-as-directory later:
 			w.enqueue(walkItem{dir: joined, callbackDone: true})
 			return nil
 		}
-		if err == filepath.SkipDir {
+		if errors.Is(err, filepath.SkipDir) {
 			// Permit SkipDir on symlinks too.
 			return nil
 		}
@@ -184,7 +184,7 @@ func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
 func (w *walker) walk(root string, runUserCallback bool) error {
 	if runUserCallback {
 		err := w.fn(root, os.ModeDir)
-		if err == filepath.SkipDir {
+		if errors.Is(err, filepath.SkipDir) {
 			return nil
 		}
 		if err != nil {

pkg/fs/directories.go

@@ -127,53 +127,54 @@ func Dirs(root string, recursive bool, followLinks bool) (result []string, err e
 	result = []string{}
 	mutex := sync.Mutex{}
 
+	// Ignore hidden folders as well as those listed in an optional ".ppignore" file.
+	ignore := NewIgnoreList(PPIgnoreFilename, true, false)
+
 	symlinks := make(map[string]bool)
 	symlinksMutex := sync.Mutex{}
 
-	appendResult := func(fileName string) {
-		fileName = strings.Replace(fileName, root, "", 1)
+	// appendResult adds the relative path of a subdirectory to the results.
+	appendResult := func(dir string) {
 		mutex.Lock()
 		defer mutex.Unlock()
-		result = append(result, fileName)
+		result = append(result, strings.Replace(dir, root, "", 1))
 	}
 
-	// Ignore hidden folders as well as those listed in an optional ".ppignore" file.
-	ignore := NewIgnoreList(PPIgnoreFilename, true, false)
-	_ = ignore.Dir(root)
-
-	err = fastwalk.Walk(root, func(fileName string, typ os.FileMode) error {
-		if typ.IsDir() || typ == os.ModeSymlink && followLinks {
-			if ignore.Ignore(fileName) {
-				return filepath.SkipDir
-			} else if FileExists(filepath.Join(fileName, PPStorageFilename)) {
-				return filepath.SkipDir
-			}
-
+	err = fastwalk.Walk(root, func(dir string, mode os.FileMode) error {
+		if mode.IsDir() || mode == os.ModeSymlink && followLinks {
 			// Skip if symlink does not point to existing directory.
-			if typ == os.ModeSymlink {
-				if info, err := os.Stat(fileName); err != nil || !info.IsDir() {
+			if mode == os.ModeSymlink {
+				if info, statErr := os.Stat(dir); statErr != nil || !info.IsDir() {
 					return filepath.SkipDir
 				}
 			}
 
-			if fileName != root {
+			// Skip if directory should be ignored.
+			if _ = ignore.Path(dir); ignore.Ignore(dir) {
+				return filepath.SkipDir
+			} else if FileExists(filepath.Join(dir, PPStorageFilename)) {
+				return filepath.SkipDir
+			}
+
+			// Only add subdirectories.
+			if dir != root {
 				if !recursive {
-					appendResult(fileName)
+					appendResult(dir)
 
 					return filepath.SkipDir
-				} else if typ != os.ModeSymlink {
-					appendResult(fileName)
+				} else if mode != os.ModeSymlink {
+					appendResult(dir)
 
 					return nil
-				} else if resolved, err := Resolve(fileName); err == nil {
+				} else if resolved, resolveErr := Resolve(dir); resolveErr == nil {
 					symlinksMutex.Lock()
 					defer symlinksMutex.Unlock()
 
 					if _, ok := symlinks[resolved]; ok {
 						return filepath.SkipDir
 					} else {
 						symlinks[resolved] = true
-						appendResult(fileName)
+						appendResult(dir)
 					}
 
 					return fastwalk.ErrTraverseLink

pkg/fs/directories_test.go

@@ -15,7 +15,7 @@ func TestDirs(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		assert.Len(t, result, 9)
+		assert.Len(t, result, 8)
 		assert.Contains(t, result, "/directory")
 		assert.Contains(t, result, "/directory/subdirectory")
 		assert.Contains(t, result, "/directory/subdirectory/animals")