Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add new scanner algorithm, can be enabled with DevNewScanner config o…
…ption
- Loading branch information
Showing
11 changed files
with
750 additions
and
160 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package scanner | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/deluan/navidrome/log" | ||
| ) | ||
|
|
||
| const ( | ||
| // batchSize used for albums/artists updates | ||
| batchSize = 5 | ||
| ) | ||
|
|
||
| type refreshCallbackFunc = func(ids ...string) error | ||
|
|
||
| type flushableMap struct { | ||
| ctx context.Context | ||
| flushFunc refreshCallbackFunc | ||
| entity string | ||
| m map[string]struct{} | ||
| } | ||
|
|
||
| func newFlushableMap(ctx context.Context, entity string, flushFunc refreshCallbackFunc) *flushableMap { | ||
| return &flushableMap{ | ||
| ctx: ctx, | ||
| flushFunc: flushFunc, | ||
| entity: entity, | ||
| m: map[string]struct{}{}, | ||
| } | ||
| } | ||
|
|
||
| func (f *flushableMap) update(id string) error { | ||
| f.m[id] = struct{}{} | ||
| if len(f.m) >= batchSize { | ||
| err := f.flush() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (f *flushableMap) flush() error { | ||
| if len(f.m) == 0 { | ||
| return nil | ||
| } | ||
| var ids []string | ||
| for id := range f.m { | ||
| ids = append(ids, id) | ||
| delete(f.m, id) | ||
| } | ||
| if err := f.flushFunc(ids...); err != nil { | ||
| log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", f.entity), err) | ||
| return err | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| package scanner | ||
|
|
||
| import ( | ||
| "context" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
| "time" | ||
|
|
||
| "github.com/deluan/navidrome/consts" | ||
| "github.com/deluan/navidrome/log" | ||
| "github.com/deluan/navidrome/utils" | ||
| ) | ||
|
|
||
| type dirMap = map[string]time.Time | ||
|
|
||
| func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) { | ||
| newMap := make(map[string]time.Time) | ||
| err := loadMap(ctx, rootFolder, rootFolder, newMap) | ||
| if err != nil { | ||
| log.Error(ctx, "Error loading directory tree", err) | ||
| } | ||
| return newMap, err | ||
| } | ||
|
|
||
| func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error { | ||
| children, lastUpdated, err := loadDir(ctx, currentFolder) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| for _, c := range children { | ||
| err := loadMap(ctx, rootPath, c, dirMap) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| dir := filepath.Clean(currentFolder) | ||
| dirMap[dir] = lastUpdated | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) { | ||
| dirInfo, err := os.Stat(dirPath) | ||
| if err != nil { | ||
| log.Error(ctx, "Error stating dir", "path", dirPath, err) | ||
| return | ||
| } | ||
| lastUpdated = dirInfo.ModTime() | ||
|
|
||
| files, err := ioutil.ReadDir(dirPath) | ||
| if err != nil { | ||
| log.Error(ctx, "Error reading dir", "path", dirPath, err) | ||
| return | ||
| } | ||
| for _, f := range files { | ||
| isDir, err := isDirOrSymlinkToDir(dirPath, f) | ||
| // Skip invalid symlinks | ||
| if err != nil { | ||
| continue | ||
| } | ||
| if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) { | ||
| children = append(children, filepath.Join(dirPath, f.Name())) | ||
| } else { | ||
| if f.ModTime().After(lastUpdated) { | ||
| lastUpdated = f.ModTime() | ||
| } | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| // isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file | ||
| // system directory, or a symbolic link to a directory. Note that if the dirInfo | ||
| // is not a directory but is a symbolic link, this method will resolve by | ||
| // sending a request to the operating system to follow the symbolic link. | ||
| // Copied from github.com/karrick/godirwalk | ||
| func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) { | ||
| if dirInfo.IsDir() { | ||
| return true, nil | ||
| } | ||
| if dirInfo.Mode()&os.ModeSymlink == 0 { | ||
| return false, nil | ||
| } | ||
| // Does this symlink point to a directory? | ||
| dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name())) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| return dirInfo.IsDir(), nil | ||
| } | ||
|
|
||
| // isDirIgnored returns true if the directory represented by dirInfo contains an | ||
| // `ignore` file (named after consts.SkipScanFile) | ||
| func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool { | ||
| _, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile)) | ||
| return err == nil | ||
| } | ||
|
|
||
| // isDirReadable returns true if the directory represented by dirInfo is readable | ||
| func isDirReadable(baseDir string, dirInfo os.FileInfo) bool { | ||
| path := filepath.Join(baseDir, dirInfo.Name()) | ||
| res, err := utils.IsDirReadable(path) | ||
| if !res { | ||
| log.Debug("Warning: Skipping unreadable directory", "path", path, err) | ||
| } | ||
| return res | ||
| } |
Oops, something went wrong.