Remove tracks from DB that were deleted while Navidrome was not running. Fixes #151

model/mediafile.go

@@ -54,6 +54,7 @@ type MediaFileRepository interface {
 	GetAll(options ...QueryOptions) (MediaFiles, error)
 	FindByAlbum(albumId string) (MediaFiles, error)
 	FindByPath(path string) (MediaFiles, error)
+	FindPathsRecursively(basePath string) ([]string, error)
 	GetStarred(options ...QueryOptions) (MediaFiles, error)
 	GetRandom(options ...QueryOptions) (MediaFiles, error)
 	Search(q string, offset int, size int) (MediaFiles, error)

persistence/mediafile_repository.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 
 	. "github.com/Masterminds/squirrel"
 	"github.com/astaxie/beego/orm"
@@ -79,11 +80,11 @@ func (r mediaFileRepository) FindByAlbum(albumId string) (model.MediaFiles, erro
 	return res, err
 }
 
+// FindByPath only return mediafiles that are direct children of requested path
 func (r mediaFileRepository) FindByPath(path string) (model.MediaFiles, error) {
-	// Only return mediafiles that are direct child of requested path
 	// Query by path based on https://stackoverflow.com/a/13911906/653632
 	sel0 := r.selectMediaFile().Columns(fmt.Sprintf("substr(path, %d) AS item", len(path)+2)).
-		Where(Like{"path": path + string(os.PathSeparator) + "%"})
+		Where(Like{"path": filepath.Join(path, "%")})
 	sel := r.newSelect().Columns("*", "item NOT GLOB '*"+string(os.PathSeparator)+"*' AS isLast").
 		Where(Eq{"isLast": 1}).FromSelect(sel0, "sel0")
 
@@ -92,6 +93,16 @@ func (r mediaFileRepository) FindByPath(path string) (model.MediaFiles, error) {
 	return res, err
 }
 
+// FindPathsRecursively returns a list of all subfolders of basePath, recursively
+func (r mediaFileRepository) FindPathsRecursively(basePath string) ([]string, error) {
+	// Query based on https://stackoverflow.com/a/38330814/653632
+	sel := r.newSelect().Columns("distinct rtrim(path, replace(path, '/', ''))").
+		Where(Like{"path": filepath.Join(basePath, "%")})
+	var res []string
+	err := r.queryAll(sel, &res)
+	return res, err
+}
+
 func (r mediaFileRepository) GetStarred(options ...model.QueryOptions) (model.MediaFiles, error) {
 	sq := r.selectMediaFile(options...).Where("starred = true")
 	starred := model.MediaFiles{}
@@ -112,9 +123,11 @@ func (r mediaFileRepository) Delete(id string) error {
 	return r.delete(Eq{"id": id})
 }
 
+// DeleteByPath delete from the DB all mediafiles that are direct children of path
 func (r mediaFileRepository) DeleteByPath(path string) error {
+	path = filepath.Clean(path)
 	del := Delete(r.tableName).
-		Where(And{Like{"path": path + string(os.PathSeparator) + "%"},
+		Where(And{Like{"path": filepath.Join(path, "%")},
 			Eq{fmt.Sprintf("substr(path, %d) glob '*%s*'", len(path)+2, string(os.PathSeparator)): 0}})
 	log.Debug(r.ctx, "Deleting mediafiles by path", "path", path)
 	_, err := r.executeSQL(del)

scanner/change_detector.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/deluan/navidrome/consts"
 	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/utils"
 )
 
 type dirInfo struct {
@@ -104,15 +105,11 @@ func isDirIgnored(baseDir string, dirInfo os.FileInfo) bool {
 // isDirReadable returns true if the directory represented by dirInfo is readable
 func isDirReadable(baseDir string, dirInfo os.FileInfo) bool {
 	path := filepath.Join(baseDir, dirInfo.Name())
-	dir, err := os.Open(path)
+	res, err := utils.IsDirReadable(path)
-	if err != nil {
+	if !res {
 		log.Debug("Warning: Skipping unreadable directory", "path", path, err)
-		return false
 	}
-	if err := dir.Close(); err != nil {
+	return res
-		log.Error("Error closing directory", "path", path, err)
-	}
-	return true
 }
 
 func (s *ChangeDetector) loadMap(dirMap dirInfoMap, path string, since time.Time, maybe bool) error {

scanner/tag_scanner.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/deluan/navidrome/consts"
@@ -21,6 +22,7 @@ type TagScanner struct {
 	rootFolder string
 	ds         model.DataStore
 	detector   *ChangeDetector
+	firstRun   sync.Once
 }
 
 func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
@@ -28,6 +30,7 @@ func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner {
 		rootFolder: rootFolder,
 		ds:         ds,
 		detector:   NewChangeDetector(rootFolder),
+		firstRun:   sync.Once{},
 	}
 }
 
@@ -45,11 +48,13 @@ const (
 )
 
 // Scan algorithm overview:
-// For each changed: Get all files from DB that starts with the folder, scan each file:
+// For each changed folder: Get all files from DB that starts with the folder, scan each file:
 //	    if file in folder is newer, update the one in DB
 //      if file in folder does not exists in DB, add
 // 	    for each file in the DB that is not found in the folder, delete from DB
 // For each deleted folder: delete all files from DB that starts with the folder path
+// Only on first run, check if any folder under each changed folder is missing.
+//      if it is, delete everything under it
 // Create new albums/artists, update counters:
 //      collect all albumIDs and artistIDs from previous steps
 //	    refresh the collected albums and artists with the metadata from the mediafiles
@@ -102,6 +107,10 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro
 		return err
 	}
 
+	s.firstRun.Do(func() {
+		s.removeDeletedFolders(context.TODO(), changed)
+	})
+
 	err = s.ds.GC(log.NewContext(context.TODO()))
 	log.Info("Finished Music Folder", "folder", s.rootFolder, "elapsed", time.Since(start))
 
@@ -270,6 +279,28 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedA
 	return s.ds.MediaFile(ctx).DeleteByPath(dir)
 }
 
+func (s *TagScanner) removeDeletedFolders(ctx context.Context, changed []string) {
+	for _, dir := range changed {
+		fullPath := filepath.Join(s.rootFolder, dir)
+		paths, err := s.ds.MediaFile(ctx).FindPathsRecursively(fullPath)
+		if err != nil {
+			log.Error(ctx, "Error reading paths from DB", "path", dir, err)
+			return
+		}
+
+		// If a path is unreadable, remove from the DB
+		for _, path := range paths {
+			if readable, err := utils.IsDirReadable(path); !readable {
+				log.Warn(ctx, "Path unavailable. Removing tracks from DB", "path", path, err)
+				err = s.ds.MediaFile(ctx).DeleteByPath(path)
+				if err != nil {
+					log.Error(ctx, "Error removing MediaFiles from DB", "path", path, err)
+				}
+			}
+		}
+	}
+}
+
 func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
 	mds, err := ExtractAllMetadata(filePaths)
 	if err != nil {

utils/paths.go

@@ -0,0 +1,18 @@
+package utils
+
+import (
+	"os"
+
+	"github.com/deluan/navidrome/log"
+)
+
+func IsDirReadable(path string) (bool, error) {
+	dir, err := os.Open(path)
+	if err != nil {
+		return false, err
+	}
+	if err := dir.Close(); err != nil {
+		log.Error("Error closing directory", "path", path, err)
+	}
+	return true, nil
+}