From e73b71aaf79e45142057132193258310ce28e58c Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 12 Jun 2020 13:26:46 -0400 Subject: [PATCH] Remove tracks from DB that were deleted while Navidrome was not running. Fixes #151 --- model/mediafile.go | 1 + persistence/mediafile_repository.go | 19 ++++++++++++++--- scanner/change_detector.go | 11 ++++------ scanner/tag_scanner.go | 33 ++++++++++++++++++++++++++++- utils/paths.go | 18 ++++++++++++++++ 5 files changed, 71 insertions(+), 11 deletions(-) create mode 100644 utils/paths.go diff --git a/model/mediafile.go b/model/mediafile.go index 35dcbbd02..ee53dd187 100644 --- a/model/mediafile.go +++ b/model/mediafile.go @@ -54,6 +54,7 @@ type MediaFileRepository interface { GetAll(options ...QueryOptions) (MediaFiles, error) FindByAlbum(albumId string) (MediaFiles, error) FindByPath(path string) (MediaFiles, error) + FindPathsRecursively(basePath string) ([]string, error) GetStarred(options ...QueryOptions) (MediaFiles, error) GetRandom(options ...QueryOptions) (MediaFiles, error) Search(q string, offset int, size int) (MediaFiles, error) diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index 6b0ac4129..1f2b935c9 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" . "github.com/Masterminds/squirrel" "github.com/astaxie/beego/orm" @@ -79,11 +80,11 @@ func (r mediaFileRepository) FindByAlbum(albumId string) (model.MediaFiles, erro return res, err } +// FindByPath only return mediafiles that are direct children of requested path func (r mediaFileRepository) FindByPath(path string) (model.MediaFiles, error) { - // Only return mediafiles that are direct child of requested path // Query by path based on https://stackoverflow.com/a/13911906/653632 sel0 := r.selectMediaFile().Columns(fmt.Sprintf("substr(path, %d) AS item", len(path)+2)). - Where(Like{"path": path + string(os.PathSeparator) + "%"}) + Where(Like{"path": filepath.Join(path, "%")}) sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast"). Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0") @@ -92,6 +93,16 @@ func (r mediaFileRepository) FindByPath(path string) (model.MediaFiles, error) { return res, err } +// FindPathsRecursively returns a list of all subfolders of basePath, recursively +func (r mediaFileRepository) FindPathsRecursively(basePath string) ([]string, error) { + // Query based on https://stackoverflow.com/a/38330814/653632 + sel := r.newSelect().Columns("distinct rtrim(path, replace(path, '/', ''))"). + Where(Like{"path": filepath.Join(basePath, "%")}) + var res []string + err := r.queryAll(sel, &res) + return res, err +} + func (r mediaFileRepository) GetStarred(options ...model.QueryOptions) (model.MediaFiles, error) { sq := r.selectMediaFile(options...).Where("starred = true") starred := model.MediaFiles{} @@ -112,9 +123,11 @@ func (r mediaFileRepository) Delete(id string) error { return r.delete(Eq{"id": id}) } +// DeleteByPath delete from the DB all mediafiles that are direct children of path func (r mediaFileRepository) DeleteByPath(path string) error { + path = filepath.Clean(path) del := Delete(r.tableName). - Where(And{Like{"path": path + string(os.PathSeparator) + "%"}, + Where(And{Like{"path": filepath.Join(path, "%")}, Eq{fmt.Sprintf("substr(path, %d) glob '*%s*'", len(path)+2, string(os.PathSeparator)): 0}}) log.Debug(r.ctx, "Deleting mediafiles by path", "path", path) _, err := r.executeSQL(del) diff --git a/scanner/change_detector.go b/scanner/change_detector.go index 66ebbea0c..342eb632a 100644 --- a/scanner/change_detector.go +++ b/scanner/change_detector.go @@ -8,6 +8,7 @@ import ( "github.com/deluan/navidrome/consts" "github.com/deluan/navidrome/log" + "github.com/deluan/navidrome/utils" ) type dirInfo struct { @@ -104,15 +105,11 @@ func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool { // isDirReadable returns true if the directory represented by dirInfo is readable func isDirReadable(baseDir string, dirInfo os.FileInfo) bool { path := filepath.Join(baseDir, dirInfo.Name()) - dir, err := os.Open(path) - if err != nil { + res, err := utils.IsDirReadable(path) + if !res { log.Debug("Warning: Skipping unreadable directory", "path", path, err) - return false } - if err := dir.Close(); err != nil { - log.Error("Error closing directory", "path", path, err) - } - return true + return res } func (s *ChangeDetector) loadMap(dirMap dirInfoMap, path string, since time.Time, maybe bool) error { diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index 40aec5bf3..e69aa160f 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sort" "strings" + "sync" "time" "github.com/deluan/navidrome/consts" @@ -21,6 +22,7 @@ type TagScanner struct { rootFolder string ds model.DataStore detector *ChangeDetector + firstRun sync.Once } func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner { @@ -28,6 +30,7 @@ func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner { rootFolder: rootFolder, ds: ds, detector: NewChangeDetector(rootFolder), + firstRun: sync.Once{}, } } @@ -45,11 +48,13 @@ const ( ) // Scan algorithm overview: -// For each changed: Get all files from DB that starts with the folder, scan each file: +// For each changed folder: Get all files from DB that starts with the folder, scan each file: // if file in folder is newer, update the one in DB // if file in folder does not exists in DB, add // for each file in the DB that is not found in the folder, delete from DB // For each deleted folder: delete all files from DB that starts with the folder path +// Only on first run, check if any folder under each changed folder is missing. +// if it is, delete everything under it // Create new albums/artists, update counters: // collect all albumIDs and artistIDs from previous steps // refresh the collected albums and artists with the metadata from the mediafiles @@ -102,6 +107,10 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro return err } + s.firstRun.Do(func() { + s.removeDeletedFolders(context.TODO(), changed) + }) + err = s.ds.GC(log.NewContext(context.TODO())) log.Info("Finished Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start)) @@ -270,6 +279,28 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedA return s.ds.MediaFile(ctx).DeleteByPath(dir) } +func (s *TagScanner) removeDeletedFolders(ctx context.Context, changed []string) { + for _, dir := range changed { + fullPath := filepath.Join(s.rootFolder, dir) + paths, err := s.ds.MediaFile(ctx).FindPathsRecursively(fullPath) + if err != nil { + log.Error(ctx, "Error reading paths from DB", "path", dir, err) + return + } + + // If a path is unreadable, remove from the DB + for _, path := range paths { + if readable, err := utils.IsDirReadable(path); !readable { + log.Warn(ctx, "Path unavailable. Removing tracks from DB", "path", path, err) + err = s.ds.MediaFile(ctx).DeleteByPath(path) + if err != nil { + log.Error(ctx, "Error removing MediaFiles from DB", "path", path, err) + } + } + } + } +} + func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) { mds, err := ExtractAllMetadata(filePaths) if err != nil { diff --git a/utils/paths.go b/utils/paths.go new file mode 100644 index 000000000..84a45eb98 --- /dev/null +++ b/utils/paths.go @@ -0,0 +1,18 @@ +package utils + +import ( + "os" + + "github.com/deluan/navidrome/log" +) + +func IsDirReadable(path string) (bool, error) { + dir, err := os.Open(path) + if err != nil { + return false, err + } + if err := dir.Close(); err != nil { + log.Error("Error closing directory", "path", path, err) + } + return true, nil +}