From 51c295d1de767355f3108436dabbe32e8aa1a0f3 Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 17 Jul 2020 10:27:30 -0400 Subject: [PATCH] Add new scanner algorithm, can be enabled with DevNewScanner config option --- conf/configuration.go | 2 + scanner/change_detector.go | 55 +----- scanner/change_detector_test.go | 8 +- scanner/flushable_map.go | 58 +++++++ scanner/load_tree.go | 109 ++++++++++++ scanner/mapping.go | 120 +++++++++++++ scanner/scanner.go | 10 +- scanner/tag_scanner.go | 113 +----------- scanner/tag_scanner_2.go | 297 ++++++++++++++++++++++++++++++++ utils/strings.go | 16 ++ utils/strings_test.go | 122 +++++++++++++ 11 files changed, 750 insertions(+), 160 deletions(-) create mode 100644 scanner/flushable_map.go create mode 100644 scanner/load_tree.go create mode 100644 scanner/mapping.go create mode 100644 scanner/tag_scanner_2.go diff --git a/conf/configuration.go b/conf/configuration.go index 01172399af1..8b81af71c4a 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -38,6 +38,7 @@ type configOptions struct { // DevFlags. These are used to enable/disable debugging and incomplete features DevLogSourceLine bool DevAutoCreateAdminPassword string + DevNewScanner bool } var Server = &configOptions{} @@ -94,6 +95,7 @@ func init() { // DevFlags. These are used to enable/disable debugging and incomplete features viper.SetDefault("devlogsourceline", false) viper.SetDefault("devautocreateadminpassword", "") + viper.SetDefault("devnewscanner", false) } func InitConfig(cfgFile string) { diff --git a/scanner/change_detector.go b/scanner/change_detector.go index 87236bcc487..2ec943290ba 100644 --- a/scanner/change_detector.go +++ b/scanner/change_detector.go @@ -7,9 +7,7 @@ import ( "path/filepath" "time" - "github.com/deluan/navidrome/consts" "github.com/deluan/navidrome/log" - "github.com/deluan/navidrome/utils" ) type dirInfo struct { @@ -18,19 +16,19 @@ type dirInfo struct { } type dirInfoMap map[string]dirInfo -type ChangeDetector struct { +type changeDetector struct { rootFolder string dirMap dirInfoMap } -func NewChangeDetector(rootFolder string) *ChangeDetector { - return &ChangeDetector{ +func newChangeDetector(rootFolder string) *changeDetector { + return &changeDetector{ rootFolder: rootFolder, dirMap: dirInfoMap{}, } } -func (s *ChangeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) { +func (s *changeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) (changed []string, deleted []string, err error) { start := time.Now() newMap := make(dirInfoMap) err = s.loadMap(ctx, newMap, s.rootFolder, lastModifiedSince, false) @@ -48,7 +46,7 @@ func (s *ChangeDetector) Scan(ctx context.Context, lastModifiedSince time.Time) return } -func (s *ChangeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) { +func (s *changeDetector) loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) { dirInfo, err := os.Stat(dirPath) if err != nil { log.Error(ctx, "Error stating dir", "path", dirPath, err) @@ -78,44 +76,7 @@ func (s *ChangeDetector) loadDir(ctx context.Context, dirPath string) (children return } -// isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file -// system directory, or a symbolic link to a directory. Note that if the dirInfo -// is not a directory but is a symbolic link, this method will resolve by -// sending a request to the operating system to follow the symbolic link. -// Copied from github.com/karrick/godirwalk -func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) { - if dirInfo.IsDir() { - return true, nil - } - if dirInfo.Mode()&os.ModeSymlink == 0 { - return false, nil - } - // Does this symlink point to a directory? - dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name())) - if err != nil { - return false, err - } - return dirInfo.IsDir(), nil -} - -// isDirIgnored returns true if the directory represented by dirInfo contains an -// `ignore` file (named after consts.SkipScanFile) -func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool { - _, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile)) - return err == nil -} - -// isDirReadable returns true if the directory represented by dirInfo is readable -func isDirReadable(baseDir string, dirInfo os.FileInfo) bool { - path := filepath.Join(baseDir, dirInfo.Name()) - res, err := utils.IsDirReadable(path) - if !res { - log.Debug("Warning: Skipping unreadable directory", "path", path, err) - } - return res -} - -func (s *ChangeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error { +func (s *changeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path string, since time.Time, maybe bool) error { children, lastUpdated, err := s.loadDir(ctx, path) if err != nil { return err @@ -134,7 +95,7 @@ func (s *ChangeDetector) loadMap(ctx context.Context, dirMap dirInfoMap, path st return nil } -func (s *ChangeDetector) getRelativePath(subFolder string) string { +func (s *changeDetector) getRelativePath(subFolder string) string { dir, _ := filepath.Rel(s.rootFolder, subFolder) if dir == "" { dir = "." @@ -142,7 +103,7 @@ func (s *ChangeDetector) getRelativePath(subFolder string) string { return dir } -func (s *ChangeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) { +func (s *changeDetector) checkForUpdates(lastModifiedSince time.Time, newMap dirInfoMap) (changed []string, deleted []string, err error) { for dir, newEntry := range newMap { lastUpdated := newEntry.mdate oldLastUpdated := lastModifiedSince diff --git a/scanner/change_detector_test.go b/scanner/change_detector_test.go index 3b7c294fe61..e12889acb61 100644 --- a/scanner/change_detector_test.go +++ b/scanner/change_detector_test.go @@ -11,9 +11,9 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("ChangeDetector", func() { +var _ = Describe("changeDetector", func() { var testFolder string - var scanner *ChangeDetector + var scanner *changeDetector lastModifiedSince := time.Time{} @@ -23,7 +23,7 @@ var _ = Describe("ChangeDetector", func() { if err != nil { panic(err) } - scanner = NewChangeDetector(testFolder) + scanner = newChangeDetector(testFolder) }) It("detects changes recursively", func() { @@ -97,7 +97,7 @@ var _ = Describe("ChangeDetector", func() { // Only returns changes after lastModifiedSince lastModifiedSince = nowWithDelay() - newScanner := NewChangeDetector(testFolder) + newScanner := newChangeDetector(testFolder) changed, deleted, err = newScanner.Scan(context.TODO(), lastModifiedSince) Expect(err).To(BeNil()) Expect(deleted).To(BeEmpty()) diff --git a/scanner/flushable_map.go b/scanner/flushable_map.go new file mode 100644 index 00000000000..4fad754dba3 --- /dev/null +++ b/scanner/flushable_map.go @@ -0,0 +1,58 @@ +package scanner + +import ( + "context" + "fmt" + + "github.com/deluan/navidrome/log" +) + +const ( + // batchSize used for albums/artists updates + batchSize = 5 +) + +type refreshCallbackFunc = func(ids ...string) error + +type flushableMap struct { + ctx context.Context + flushFunc refreshCallbackFunc + entity string + m map[string]struct{} +} + +func newFlushableMap(ctx context.Context, entity string, flushFunc refreshCallbackFunc) *flushableMap { + return &flushableMap{ + ctx: ctx, + flushFunc: flushFunc, + entity: entity, + m: map[string]struct{}{}, + } +} + +func (f *flushableMap) update(id string) error { + f.m[id] = struct{}{} + if len(f.m) >= batchSize { + err := f.flush() + if err != nil { + return err + } + } + return nil +} + +func (f *flushableMap) flush() error { + if len(f.m) == 0 { + return nil + } + var ids []string + for id := range f.m { + ids = append(ids, id) + delete(f.m, id) + } + if err := f.flushFunc(ids...); err != nil { + log.Error(f.ctx, fmt.Sprintf("Error writing %ss to the DB", f.entity), err) + return err + } + return nil +} diff --git a/scanner/load_tree.go b/scanner/load_tree.go new file mode 100644 index 00000000000..668acfa8f76 --- /dev/null +++ b/scanner/load_tree.go @@ -0,0 +1,109 @@ +package scanner + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/deluan/navidrome/consts" + "github.com/deluan/navidrome/log" + "github.com/deluan/navidrome/utils" +) + +type dirMap = map[string]time.Time + +func loadDirTree(ctx context.Context, rootFolder string) (dirMap, error) { + newMap := make(map[string]time.Time) + err := loadMap(ctx, rootFolder, rootFolder, newMap) + if err != nil { + log.Error(ctx, "Error loading directory tree", err) + } + return newMap, err +} + +func loadMap(ctx context.Context, rootPath string, currentFolder string, dirMap dirMap) error { + children, lastUpdated, err := loadDir(ctx, currentFolder) + if err != nil { + return err + } + for _, c := range children { + err := loadMap(ctx, rootPath, c, dirMap) + if err != nil { + return err + } + } + + dir := filepath.Clean(currentFolder) + dirMap[dir] = lastUpdated + + return nil +} + +func loadDir(ctx context.Context, dirPath string) (children []string, lastUpdated time.Time, err error) { + dirInfo, err := os.Stat(dirPath) + if err != nil { + log.Error(ctx, "Error stating dir", "path", dirPath, err) + return + } + lastUpdated = dirInfo.ModTime() + + files, err := ioutil.ReadDir(dirPath) + if err != nil { + log.Error(ctx, "Error reading dir", "path", dirPath, err) + return + } + for _, f := range files { + isDir, err := isDirOrSymlinkToDir(dirPath, f) + // Skip invalid symlinks + if err != nil { + continue + } + if isDir && !isDirIgnored(dirPath, f) && isDirReadable(dirPath, f) { + children = append(children, filepath.Join(dirPath, f.Name())) + } else { + if f.ModTime().After(lastUpdated) { + lastUpdated = f.ModTime() + } + } + } + return +} + +// isDirOrSymlinkToDir returns true if and only if the dirInfo represents a file +// system directory, or a symbolic link to a directory. Note that if the dirInfo +// is not a directory but is a symbolic link, this method will resolve by +// sending a request to the operating system to follow the symbolic link. +// Copied from github.com/karrick/godirwalk +func isDirOrSymlinkToDir(baseDir string, dirInfo os.FileInfo) (bool, error) { + if dirInfo.IsDir() { + return true, nil + } + if dirInfo.Mode()&os.ModeSymlink == 0 { + return false, nil + } + // Does this symlink point to a directory? + dirInfo, err := os.Stat(filepath.Join(baseDir, dirInfo.Name())) + if err != nil { + return false, err + } + return dirInfo.IsDir(), nil +} + +// isDirIgnored returns true if the directory represented by dirInfo contains an +// `ignore` file (named after consts.SkipScanFile) +func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool { + _, err := os.Stat(filepath.Join(baseDir, dirInfo.Name(), consts.SkipScanFile)) + return err == nil +} + +// isDirReadable returns true if the directory represented by dirInfo is readable +func isDirReadable(baseDir string, dirInfo os.FileInfo) bool { + path := filepath.Join(baseDir, dirInfo.Name()) + res, err := utils.IsDirReadable(path) + if !res { + log.Debug("Warning: Skipping unreadable directory", "path", path, err) + } + return res +} diff --git a/scanner/mapping.go b/scanner/mapping.go new file mode 100644 index 00000000000..deda3e46f73 --- /dev/null +++ b/scanner/mapping.go @@ -0,0 +1,120 @@ +package scanner + +import ( + "crypto/md5" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/deluan/navidrome/consts" + "github.com/deluan/navidrome/model" + "github.com/deluan/navidrome/utils" + "github.com/kennygrant/sanitize" +) + +type mediaFileMapper struct { + rootFolder string +} + +func newMediaFileMapper(rootFolder string) *mediaFileMapper { + return &mediaFileMapper{rootFolder: rootFolder} +} + +func (s *mediaFileMapper) toMediaFile(md *Metadata) model.MediaFile { + mf := &model.MediaFile{} + mf.ID = s.trackID(md) + mf.Title = s.mapTrackTitle(md) + mf.Album = md.Album() + mf.AlbumID = s.albumID(md) + mf.Album = s.mapAlbumName(md) + mf.ArtistID = s.artistID(md) + mf.Artist = s.mapArtistName(md) + mf.AlbumArtistID = s.albumArtistID(md) + mf.AlbumArtist = s.mapAlbumArtistName(md) + mf.Genre = md.Genre() + mf.Compilation = md.Compilation() + mf.Year = md.Year() + mf.TrackNumber, _ = md.TrackNumber() + mf.DiscNumber, _ = md.DiscNumber() + mf.DiscSubtitle = md.DiscSubtitle() + mf.Duration = md.Duration() + mf.BitRate = md.BitRate() + mf.Path = md.FilePath() + mf.Suffix = md.Suffix() + mf.Size = md.Size() + mf.HasCoverArt = md.HasPicture() + mf.SortTitle = md.SortTitle() + mf.SortAlbumName = md.SortAlbum() + mf.SortArtistName = md.SortArtist() + mf.SortAlbumArtistName = md.SortAlbumArtist() + mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album) + mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist) + mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist) + + // TODO Get Creation time. https://github.com/djherbis/times ? + mf.CreatedAt = md.ModificationTime() + mf.UpdatedAt = md.ModificationTime() + + return *mf +} + +func sanitizeFieldForSorting(originalValue string) string { + v := utils.NoArticle(originalValue) + v = strings.TrimSpace(sanitize.Accents(v)) + return utils.NoArticle(v) +} + +func (s *mediaFileMapper) mapTrackTitle(md *Metadata) string { + if md.Title() == "" { + s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator)) + e := filepath.Ext(s) + return strings.TrimSuffix(s, e) + } + return md.Title() +} + +func (s *mediaFileMapper) mapAlbumArtistName(md *Metadata) string { + switch { + case md.Compilation(): + return consts.VariousArtists + case md.AlbumArtist() != "": + return md.AlbumArtist() + case md.Artist() != "": + return md.Artist() + default: + return consts.UnknownArtist + } +} + +func (s *mediaFileMapper) mapArtistName(md *Metadata) string { + if md.Artist() != "" { + return md.Artist() + } + return consts.UnknownArtist +} + +func (s *mediaFileMapper) mapAlbumName(md *Metadata) string { + name := md.Album() + if name == "" { + return "[Unknown Album]" + } + return name +} + +func (s *mediaFileMapper) trackID(md *Metadata) string { + return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath()))) +} + +func (s *mediaFileMapper) albumID(md *Metadata) string { + albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md))) + return fmt.Sprintf("%x", md5.Sum([]byte(albumPath))) +} + +func (s *mediaFileMapper) artistID(md *Metadata) string { + return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md))))) +} + +func (s *mediaFileMapper) albumArtistID(md *Metadata) string { + return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md))))) +} diff --git a/scanner/scanner.go b/scanner/scanner.go index 1ebade79749..92d360b4d1b 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -7,6 +7,7 @@ import ( "strconv" "time" + "github.com/deluan/navidrome/conf" "github.com/deluan/navidrome/log" "github.com/deluan/navidrome/model" ) @@ -81,10 +82,17 @@ func (s *Scanner) loadFolders() { fs, _ := s.ds.MediaFolder(context.TODO()).GetAll() for _, f := range fs { log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path) - s.folders[f.Path] = NewTagScanner(f.Path, s.ds) + s.folders[f.Path] = s.newScanner(f) } } +func (s *Scanner) newScanner(f model.MediaFolder) FolderScanner { + if conf.Server.DevNewScanner { + return NewTagScanner2(f.Path, s.ds) + } + return NewTagScanner(f.Path, s.ds) +} + type Status int type StatusInfo struct { diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index 59308168fe5..0b5ebea42a9 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -2,8 +2,6 @@ package scanner import ( "context" - "crypto/md5" - "fmt" "os" "path/filepath" "sort" @@ -11,17 +9,16 @@ import ( "sync" "time" - "github.com/deluan/navidrome/consts" "github.com/deluan/navidrome/log" "github.com/deluan/navidrome/model" "github.com/deluan/navidrome/utils" - "github.com/kennygrant/sanitize" ) type TagScanner struct { rootFolder string ds model.DataStore - detector *ChangeDetector + detector *changeDetector + mapper *mediaFileMapper firstRun sync.Once } @@ -29,7 +26,8 @@ func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner { return &TagScanner{ rootFolder: rootFolder, ds: ds, - detector: NewChangeDetector(rootFolder), + detector: newChangeDetector(rootFolder), + mapper: newMediaFileMapper(rootFolder), firstRun: sync.Once{}, } } @@ -46,9 +44,6 @@ type ( ) const ( - // batchSize used for albums/artists updates - batchSize = 5 - // filesBatchSize used for extract file metadata filesBatchSize = 100 ) @@ -339,110 +334,12 @@ func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) { var mfs model.MediaFiles for _, md := range mds { - mf := s.toMediaFile(md) + mf := s.mapper.toMediaFile(md) mfs = append(mfs, mf) } return mfs, nil } -func (s *TagScanner) toMediaFile(md *Metadata) model.MediaFile { - mf := &model.MediaFile{} - mf.ID = s.trackID(md) - mf.Title = s.mapTrackTitle(md) - mf.Album = md.Album() - mf.AlbumID = s.albumID(md) - mf.Album = s.mapAlbumName(md) - mf.ArtistID = s.artistID(md) - mf.Artist = s.mapArtistName(md) - mf.AlbumArtistID = s.albumArtistID(md) - mf.AlbumArtist = s.mapAlbumArtistName(md) - mf.Genre = md.Genre() - mf.Compilation = md.Compilation() - mf.Year = md.Year() - mf.TrackNumber, _ = md.TrackNumber() - mf.DiscNumber, _ = md.DiscNumber() - mf.DiscSubtitle = md.DiscSubtitle() - mf.Duration = md.Duration() - mf.BitRate = md.BitRate() - mf.Path = md.FilePath() - mf.Suffix = md.Suffix() - mf.Size = md.Size() - mf.HasCoverArt = md.HasPicture() - mf.SortTitle = md.SortTitle() - mf.SortAlbumName = md.SortAlbum() - mf.SortArtistName = md.SortArtist() - mf.SortAlbumArtistName = md.SortAlbumArtist() - mf.OrderAlbumName = sanitizeFieldForSorting(mf.Album) - mf.OrderArtistName = sanitizeFieldForSorting(mf.Artist) - mf.OrderAlbumArtistName = sanitizeFieldForSorting(mf.AlbumArtist) - - // TODO Get Creation time. https://github.com/djherbis/times ? - mf.CreatedAt = md.ModificationTime() - mf.UpdatedAt = md.ModificationTime() - - return *mf -} - -func sanitizeFieldForSorting(originalValue string) string { - v := utils.NoArticle(originalValue) - v = strings.TrimSpace(sanitize.Accents(v)) - return utils.NoArticle(v) -} - -func (s *TagScanner) mapTrackTitle(md *Metadata) string { - if md.Title() == "" { - s := strings.TrimPrefix(md.FilePath(), s.rootFolder+string(os.PathSeparator)) - e := filepath.Ext(s) - return strings.TrimSuffix(s, e) - } - return md.Title() -} - -func (s *TagScanner) mapAlbumArtistName(md *Metadata) string { - switch { - case md.Compilation(): - return consts.VariousArtists - case md.AlbumArtist() != "": - return md.AlbumArtist() - case md.Artist() != "": - return md.Artist() - default: - return consts.UnknownArtist - } -} - -func (s *TagScanner) mapArtistName(md *Metadata) string { - if md.Artist() != "" { - return md.Artist() - } - return consts.UnknownArtist -} - -func (s *TagScanner) mapAlbumName(md *Metadata) string { - name := md.Album() - if name == "" { - return "[Unknown Album]" - } - return name -} - -func (s *TagScanner) trackID(md *Metadata) string { - return fmt.Sprintf("%x", md5.Sum([]byte(md.FilePath()))) -} - -func (s *TagScanner) albumID(md *Metadata) string { - albumPath := strings.ToLower(fmt.Sprintf("%s\\%s", s.mapAlbumArtistName(md), s.mapAlbumName(md))) - return fmt.Sprintf("%x", md5.Sum([]byte(albumPath))) -} - -func (s *TagScanner) artistID(md *Metadata) string { - return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapArtistName(md))))) -} - -func (s *TagScanner) albumArtistID(md *Metadata) string { - return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(s.mapAlbumArtistName(md))))) -} - func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) { dir, err := os.Open(dirPath) if err != nil { diff --git a/scanner/tag_scanner_2.go b/scanner/tag_scanner_2.go new file mode 100644 index 00000000000..dc756e096e1 --- /dev/null +++ b/scanner/tag_scanner_2.go @@ -0,0 +1,297 @@ +package scanner + +import ( + "context" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/deluan/navidrome/log" + "github.com/deluan/navidrome/model" + "github.com/deluan/navidrome/utils" +) + +type TagScanner2 struct { + rootFolder string + ds model.DataStore + mapper *mediaFileMapper + albumMap *flushableMap + artistMap *flushableMap + cnt *counters +} + +func NewTagScanner2(rootFolder string, ds model.DataStore) *TagScanner2 { + return &TagScanner2{ + rootFolder: rootFolder, + mapper: newMediaFileMapper(rootFolder), + ds: ds, + } +} + +// Scan algorithm overview: +// Load all directories under the music folder, with their ModTime (self or any non-dir children) +// Find changed folders (based on lastModifiedSince) and deletes folders (comparing to the DB) +// For each deleted folder: delete all files from DB whose path starts with the delete folder path +// For each changed folder: Get all files from DB whose path starts with the changed folder, scan each file: +// if file in folder is newer, update the one in DB +// if file in folder does not exists in DB, add +// for each file in the DB that is not found in the folder, delete from DB +// Create new albums/artists, update counters: +// collect all albumIDs and artistIDs from previous steps +// refresh the collected albums and artists with the metadata from the mediafiles +// Delete all empty albums, delete all empty Artists +func (s *TagScanner2) Scan(ctx context.Context, lastModifiedSince time.Time) error { + start := time.Now() + allDirs, err := s.getDirTree(ctx) + if err != nil { + return err + } + + changedDirs := s.getChangedDirs(ctx, allDirs, lastModifiedSince) + if len(changedDirs) == 0 { + log.Debug(ctx, "No changes found in Music Folder", "folder", s.rootFolder) + return nil + } + deletedDirs, _ := s.getDeletedDirs(ctx, allDirs, changedDirs) + + if log.CurrentLevel() >= log.LevelTrace { + log.Info(ctx, "Folder changes found", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs), + "changed", strings.Join(changedDirs, ";"), "deleted", strings.Join(deletedDirs, ";")) + } else { + log.Info(ctx, "Folder changes found", "changedFolders", len(changedDirs), "deletedFolders", len(deletedDirs)) + } + + s.albumMap = newFlushableMap(ctx, "album", s.ds.Album(ctx).Refresh) + s.artistMap = newFlushableMap(ctx, "artist", s.ds.Artist(ctx).Refresh) + s.cnt = &counters{} + + for _, dir := range deletedDirs { + err := s.processDeletedDir(ctx, dir) + if err != nil { + log.Error("Error removing deleted folder from DB", "path", dir, err) + continue + } + } + for _, dir := range changedDirs { + err := s.processChangedDir(ctx, dir) + if err != nil { + log.Error("Error updating folder in the DB", "path", dir, err) + continue + } + } + + _ = s.albumMap.flush() + _ = s.artistMap.flush() + + err = s.ds.GC(log.NewContext(ctx)) + log.Info("Finished processing Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start), + "added", s.cnt.added, "updated", s.cnt.updated, "deleted", s.cnt.deleted) + + return err +} + +func (s *TagScanner2) getDirTree(ctx context.Context) (dirMap, error) { + start := time.Now() + log.Trace(ctx, "Loading directory tree from music folder", "folder", s.rootFolder) + dirs, err := loadDirTree(ctx, s.rootFolder) + if err != nil { + return nil, err + } + log.Trace("Directory tree loaded", "total", len(dirs), "elapsed", time.Since(start)) + return dirs, nil +} + +func (s *TagScanner2) getChangedDirs(ctx context.Context, dirs dirMap, lastModified time.Time) []string { + start := time.Now() + log.Trace(ctx, "Checking for changed folders") + var changed []string + for d, t := range dirs { + if t.After(lastModified) { + changed = append(changed, d) + } + } + sort.Strings(changed) + log.Trace(ctx, "Finished changed folders check", "total", len(changed), "elapsed", time.Since(start)) + return changed +} + +func (s *TagScanner2) getDeletedDirs(ctx context.Context, allDirs dirMap, changedDirs []string) ([]string, error) { + start := time.Now() + log.Trace(ctx, "Checking for deleted folders") + + var deleted []string + repo := s.ds.MediaFile(ctx) + + // If rootFolder is in the list of changedDirs, optimize and only do one query to the DB + var foldersToCheck []string + if utils.StringInSlice(s.rootFolder, changedDirs) { + foldersToCheck = []string{s.rootFolder} + } else { + foldersToCheck = changedDirs + } + + for _, changedDir := range foldersToCheck { + dirs, err := repo.FindPathsRecursively(changedDir) + if err != nil { + log.Error("Error getting subfolders from DB", "path", changedDir, err) + continue + } + for _, d := range dirs { + d := filepath.Clean(d) + if _, ok := allDirs[d]; !ok { + deleted = append(deleted, d) + } + } + } + + sort.Strings(deleted) + log.Trace(ctx, "Finished deleted folders check", "total", len(deleted), "elapsed", time.Since(start)) + return deleted, nil +} + +func (s *TagScanner2) processDeletedDir(ctx context.Context, dir string) error { + start := time.Now() + + mfs, err := s.ds.MediaFile(ctx).FindByPath(dir) + if err != nil { + return err + } + for _, t := range mfs { + err = s.albumMap.update(t.AlbumID) + if err != nil { + return err + } + err = s.artistMap.update(t.AlbumArtistID) + if err != nil { + return err + } + } + + log.Info(ctx, "Finished processing deleted folder", "path", dir, "purged", len(mfs), "elapsed", time.Since(start)) + c, err := s.ds.MediaFile(ctx).DeleteByPath(dir) + s.cnt.deleted += c + return err +} + +func (s *TagScanner2) processChangedDir(ctx context.Context, dir string) error { + start := time.Now() + + // Load folder's current tracks from DB into a map + currentTracks := map[string]model.MediaFile{} + ct, err := s.ds.MediaFile(ctx).FindByPath(dir) + if err != nil { + return err + } + for _, t := range ct { + currentTracks[t.Path] = t + } + + // Load tracks FileInfo from the folder + files, err := LoadAllAudioFiles(dir) + if err != nil { + return err + } + + // If no files to process, return + if len(files)+len(currentTracks) == 0 { + return nil + } + + // If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks + log.Trace(ctx, "Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files)) + var filesToUpdate []string + for filePath, info := range files { + c, ok := currentTracks[filePath] + if !ok { + filesToUpdate = append(filesToUpdate, filePath) + s.cnt.added++ + } + if ok && info.ModTime().After(c.UpdatedAt) { + filesToUpdate = append(filesToUpdate, filePath) + s.cnt.updated++ + } + delete(currentTracks, filePath) + + // Force a refresh of the album and artist, to cater for cover art files + err = s.albumMap.update(c.AlbumID) + if err != nil { + return err + } + err = s.artistMap.update(c.AlbumArtistID) + if err != nil { + return err + } + } + + numUpdatedTracks := 0 + numPurgedTracks := 0 + + if len(filesToUpdate) > 0 { + // Break the file list in chunks to avoid calling ffmpeg with too many parameters + chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize) + for _, chunk := range chunks { + // Load tracks Metadata from the folder + newTracks, err := s.loadTracks(chunk) + if err != nil { + return err + } + + // If track from folder is newer than the one in DB, update/insert in DB + log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "files", chunk, "numFiles", len(chunk)) + for i := range newTracks { + n := newTracks[i] + err := s.ds.MediaFile(ctx).Put(&n) + if err != nil { + return err + } + err = s.albumMap.update(n.AlbumID) + if err != nil { + return err + } + err = s.artistMap.update(n.AlbumArtistID) + if err != nil { + return err + } + numUpdatedTracks++ + } + } + } + + if len(currentTracks) > 0 { + log.Trace(ctx, "Deleting dangling tracks from DB", "dir", dir, "numTracks", len(currentTracks)) + // Remaining tracks from DB that are not in the folder are deleted + for _, ct := range currentTracks { + numPurgedTracks++ + err = s.albumMap.update(ct.AlbumID) + if err != nil { + return err + } + err = s.artistMap.update(ct.AlbumArtistID) + if err != nil { + return err + } + if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil { + return err + } + s.cnt.deleted++ + } + } + + log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks, "purged", numPurgedTracks, "elapsed", time.Since(start)) + return nil +} + +func (s *TagScanner2) loadTracks(filePaths []string) (model.MediaFiles, error) { + mds, err := ExtractAllMetadata(filePaths) + if err != nil { + return nil, err + } + + var mfs model.MediaFiles + for _, md := range mds { + mf := s.mapper.toMediaFile(md) + mfs = append(mfs, mf) + } + return mfs, nil +} diff --git a/utils/strings.go b/utils/strings.go index 4032aab1a60..9e57f2425df 100644 --- a/utils/strings.go +++ b/utils/strings.go @@ -52,3 +52,19 @@ func BreakUpStringSlice(mediaFileIds []string, chunkSize int) [][]string { } return chunks } + +func LongestCommonPrefix(list []string) string { + if len(list) == 0 { + return "" + } + + for l := 0; l < len(list[0]); l++ { + c := list[0][l] + for i := 1; i < len(list); i++ { + if l >= len(list[i]) || list[i][l] != c { + return list[i][0:l] + } + } + } + return list[0] +} diff --git a/utils/strings_test.go b/utils/strings_test.go index 1b947afac3a..6d18aa8a092 100644 --- a/utils/strings_test.go +++ b/utils/strings_test.go @@ -81,4 +81,126 @@ var _ = Describe("Strings", func() { Expect(chunks[1]).To(ConsistOf("d", "e")) }) }) + + Describe("LongestCommonPrefix", func() { + It("finds the longest common prefix", func() { + Expect(LongestCommonPrefix(testPaths)).To(Equal("/Music/iTunes 1/iTunes Media/Music/")) + }) + }) + }) + +var testPaths = []string{ + "/Music/iTunes 1/iTunes Media/Music/ABBA/Gold_ Greatest Hits/Dancing Queen.m4a", + "/Music/iTunes 1/iTunes Media/Music/ABBA/Gold_ Greatest Hits/Mamma Mia.m4a", + "/Music/iTunes 1/iTunes Media/Music/Art Blakey/A Night At Birdland, Vol. 1/01 Annoucement By Pee Wee Marquette.m4a", + "/Music/iTunes 1/iTunes Media/Music/Art Blakey/A Night At Birdland, Vol. 1/02 Split Kick.m4a", + "/Music/iTunes 1/iTunes Media/Music/As Frenéticas/As Frenéticas/Perigosa.m4a", + "/Music/iTunes 1/iTunes Media/Music/Bachman-Turner Overdrive/Gold/Down Down.m4a", + "/Music/iTunes 1/iTunes Media/Music/Bachman-Turner Overdrive/Gold/Hey You.m4a", + "/Music/iTunes 1/iTunes Media/Music/Bachman-Turner Overdrive/Gold/Hold Back The Water.m4a", + "/Music/iTunes 1/iTunes Media/Music/Belle And Sebastian/Write About Love/01 I Didn't See It Coming.m4a", + "/Music/iTunes 1/iTunes Media/Music/Belle And Sebastian/Write About Love/02 Come On Sister.m4a", + "/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Elephunk/03 Let's Get Retarded.m4a", + "/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Elephunk/04 Hey Mama.m4a", + "/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Monkey Business/10 They Don't Want Music (Feat. James Brown).m4a", + "/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/The E.N.D/1-01 Boom Boom Pow.m4a", + "/Music/iTunes 1/iTunes Media/Music/Black Eyed Peas/Timeless/01 Mas Que Nada.m4a", + "/Music/iTunes 1/iTunes Media/Music/Blondie/Heart Of Glass/Heart Of Glass.m4a", + "/Music/iTunes 1/iTunes Media/Music/Bob Dylan/Nashville Skyline/06 Lay Lady Lay.m4a", + "/Music/iTunes 1/iTunes Media/Music/Botany/Feeling Today - EP/03 Waterparker.m4a", + "/Music/iTunes 1/iTunes Media/Music/Céu/CéU/06 10 Contados.m4a", + "/Music/iTunes 1/iTunes Media/Music/Chance/Six Through Ten/03 Forgive+Forget.m4a", + "/Music/iTunes 1/iTunes Media/Music/Clive Tanaka Y Su Orquesta/Jet Set Siempre 1°/03 Neu Chicago (Side A) [For Dance].m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Absolute Rock Classics/1-02 Smoke on the water.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Almost Famous Soundtrack/10 Simple Man.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Audio News - Rock'n' Roll Forever/01 Rock Around The Clock.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Austin Powers_ International Man Of Mystery/01 The Magic Piper (Of Love).m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Austin Powers_ The Spy Who Shagged Me/04 American Woman.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Back To Dance/03 Long Cool Woman In A Black Dress.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Back To The 70's - O Album Da Década/03 American Pie.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Bambolê/09 In The Mood.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Bambolê - Volume II/03 Blue Moon.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Big Brother Brasil 2004/04 I Will Survive.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Collateral Soundtrack/03 Hands Of Time.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Estúpido Cupido (Internacional)/08 The Twist.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Forrest Gump - The Soundtrack/1-12 California Dreamin'.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Forrest Gump - The Soundtrack/1-16 Mrs. Robinson.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Ghost World - Original Motion Picture Soundtrack/01 Jaan Pechechaan Ho.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Grease [Original Soundtrack]/01 Grease.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/La Bamba/09 Summertime Blues.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Pretty Woman/10 Oh Pretty Woman.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents African Groove/01 Saye Mogo Bana.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Arabic Groove/02 Galbi.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Asian Groove/03 Remember Tomorrow.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/01 Midnight Dream.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/03 Banal Reality.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/04 Parchman Blues.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Blues Lounge/06 Run On.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Brazilian Groove/01 Maria Moita.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Brazilian Lounge/08 E Depois....m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Brazilian Lounge/11 Os Grilos.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/01 Un Simple Histoire.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/02 Limbe.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/05 Sempre Di Domenica.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents Euro Lounge/12 Voulez-Vous_.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents World Lounge/03 Santa Maria.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ A New Groove/02 Dirty Laundry.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ Blues Around the World/02 Canceriano Sem Lar (Clinica Tobias Blues).m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ Euro Groove/03 Check In.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Putumayo Presents_ World Groove/01 Attention.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Saturday Night Fever/01 Stayin' Alive.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/Saturday Night Fever/03 Night Fever.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/The Best Air Guitar Album In The World... Ever!/2-06 Johnny B. Goode.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/The Full Monty - Soundtrack/02 You Sexy Thing.m4a", + "/Music/iTunes 1/iTunes Media/Music/Compilations/The Full Monty - Soundtrack/11 We Are Family.m4a", + "/Music/iTunes 1/iTunes Media/Music/Cut Copy/Zonoscope (Bonus Version)/10 Corner of the Sky.m4a", + "/Music/iTunes 1/iTunes Media/Music/David Bowie/Changesbowie/07 Diamond Dogs.m4a", + "/Music/iTunes 1/iTunes Media/Music/Douster & Savage Skulls/Get Rich or High Tryin' - EP/01 Bad Gal.m4a", + "/Music/iTunes 1/iTunes Media/Music/Elton John/Greatest Hits 1970-2002/1-04 Rocket Man (I Think It's Going to Be a Long, Long Time).m4a", + "/Music/iTunes 1/iTunes Media/Music/Elvis Presley/ELV1S 30 #1 Hits/02 Don't Be Cruel.m4a", + "/Music/iTunes 1/iTunes Media/Music/Eric Clapton/The Cream Of Clapton/03 I Feel Free.m4a", + "/Music/iTunes 1/iTunes Media/Music/Fleetwood Mac/The Very Best Of Fleetwood Mac/02 Don't Stop.m4a", + "/Music/iTunes 1/iTunes Media/Music/Françoise Hardy/Comment te dire adieu/Comment te dire adieu.m4a", + "/Music/iTunes 1/iTunes Media/Music/Games/That We Can Play - EP/01 Strawberry Skies.m4a", + "/Music/iTunes 1/iTunes Media/Music/Grand Funk Railroad/Collectors Series/The Loco-Motion.m4a", + "/Music/iTunes 1/iTunes Media/Music/Henry Mancini/The Pink Panther (Music from the Film Score)/The Pink Panther Theme.m4a", + "/Music/iTunes 1/iTunes Media/Music/Holy Ghost!/Do It Again - Single/01 Do It Again.m4a", + "/Music/iTunes 1/iTunes Media/Music/K.C. & The Sunshine Band/The Best of/03 I'm Your Boogie Man.m4a", + "/Music/iTunes 1/iTunes Media/Music/K.C. & The Sunshine Band/Unknown Album/Megamix (Thats The Way, Shake Your Booty, Get Down Tonight, Give It Up).m4a", + "/Music/iTunes 1/iTunes Media/Music/Kim Ann Foxman & Andy Butler/Creature - EP/01 Creature.m4a", + "/Music/iTunes 1/iTunes Media/Music/Nico/Chelsea Girl/01 The Fairest Of The Seasons.m4a", + "/Music/iTunes 1/iTunes Media/Music/oOoOO/oOoOO - EP/02 Burnout Eyess.m4a", + "/Music/iTunes 1/iTunes Media/Music/Peter Frampton/The Very Best of Peter Frampton/Baby, I Love Your Way.m4a", + "/Music/iTunes 1/iTunes Media/Music/Peter Frampton/The Very Best of Peter Frampton/Show Me The Way.m4a", + "/Music/iTunes 1/iTunes Media/Music/Raul Seixas/A Arte De Raul Seixas/03 Metamorfose Ambulante.m4a", + "/Music/iTunes 1/iTunes Media/Music/Raul Seixas/A Arte De Raul Seixas/18 Eu Nasci há 10 Mil Anos Atrás.m4a", + "/Music/iTunes 1/iTunes Media/Music/Rick James/Street Songs/Super Freak.m4a", + "/Music/iTunes 1/iTunes Media/Music/Rita Lee/Fruto Proibido/Agora Só Falta Você.m4a", + "/Music/iTunes 1/iTunes Media/Music/Rita Lee/Fruto Proibido/Esse Tal De Roque Enrow.m4a", + "/Music/iTunes 1/iTunes Media/Music/Roberto Carlos/Roberto Carlos 1966/05 Negro Gato.m4a", + "/Music/iTunes 1/iTunes Media/Music/SOHO/Goddess/02 Hippychick.m4a", + "/Music/iTunes 1/iTunes Media/Music/Stan Getz/Getz_Gilberto/05 Corcovado (Quiet Nights of Quiet Stars).m4a", + "/Music/iTunes 1/iTunes Media/Music/Steely Dan/Pretzel Logic/Rikki Don't Loose That Number.m4a", + "/Music/iTunes 1/iTunes Media/Music/Stevie Wonder/For Once In My Life/I Don't Know Why.m4a", + "/Music/iTunes 1/iTunes Media/Music/Teebs/Ardour/While You Doooo.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Beatles/Magical Mystery Tour/08 Strawberry Fields Forever.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Beatles/Past Masters, Vol. 1/10 Long Tall Sally.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Beatles/Please Please Me/14 Twist And Shout.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Beatles/Sgt. Pepper's Lonely Hearts Club Band/03 Lucy In The Sky With Diamonds.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Black Crowes/Amorica/09 Wiser Time.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Black Crowes/By Your Side/05 Only A Fool.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Black Crowes/Shake Your Money Maker/04 Could I''ve Been So Blind.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Black Crowes/The Southern Harmony And Musical Companion/01 Sting Me.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Black Crowes/Three Snakes And One Charm/02 Good Friday.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Doors/Strange Days (40th Anniversary Mixes)/01 Strange Days.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Rolling Stones/Forty Licks/1-03 (I Can't Get No) Satisfaction.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Velvet Underground/The Velvet Underground & Nico/02 I'm Waiting For The Man.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Velvet Underground/The Velvet Underground & Nico/03 Femme Fatale.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Velvet Underground/White Light_White Heat/04 Here She Comes Now.m4a", + "/Music/iTunes 1/iTunes Media/Music/The Who/Sings My Generation/My Generation.m4a", + "/Music/iTunes 1/iTunes Media/Music/Village People/The Very Best Of Village People/Macho Man.m4a", + "/Music/iTunes 1/iTunes Media/Music/Vondelpark/Sauna - EP/01 California Analog Dream.m4a", + "/Music/iTunes 1/iTunes Media/Music/War/Why Can't We Be Friends/Low Rider.m4a", + "/Music/iTunes 1/iTunes Media/Music/Yes/Fragile/01 Roundabout.m4a", +}