diff --git a/adapters/lastfm/agent.go b/adapters/lastfm/agent.go index 2ed5e3c74..4a61baa71 100644 --- a/adapters/lastfm/agent.go +++ b/adapters/lastfm/agent.go @@ -388,10 +388,11 @@ func (l *lastfmAgent) Scrobble(ctx context.Context, userId string, s scrobbler.S return errors.Join(err, scrobbler.ErrNotAuthorized) } - if s.Duration <= 30 { + if s.Duration <= float32(conf.Server().LastFM.MinScrobbleDuration.Seconds()) { log.Debug(ctx, "Skipping Last.fm scrobble for short song", "track", s.Title, "duration", s.Duration) return nil } + err = l.client.scrobble(ctx, sk, ScrobbleInfo{ artist: l.getArtistForScrobble(&s.MediaFile, model.RoleArtist, s.Artist), track: s.Title, diff --git a/adapters/lastfm/agent_test.go b/adapters/lastfm/agent_test.go index ddfe520a9..a923e21d2 100644 --- a/adapters/lastfm/agent_test.go +++ b/adapters/lastfm/agent_test.go @@ -471,8 +471,8 @@ var _ = Describe("lastfmAgent", func() { }) }) - It("skips songs with less than 31 seconds", func() { - track.Duration = 29 + It("skips songs at the 30-second minimum", func() { + track.Duration = 30 httpClient.Res = http.Response{Body: io.NopCloser(bytes.NewBufferString("{}")), StatusCode: 200} err := agent.Scrobble(ctx, "user-1", scrobbler.Scrobble{MediaFile: *track, TimeStamp: time.Now()}) @@ -481,6 +481,46 @@ var _ = Describe("lastfmAgent", func() { Expect(httpClient.SavedRequest).To(BeNil()) }) + It("sends the duration parameter for a 30.5-second song", func() { + track.Duration = 30.5 + httpClient.Res = http.Response{Body: io.NopCloser(bytes.NewBufferString(`{"scrobbles":{"scrobble":{"ignoredMessage":{"code":"0"}},"@attr":{"accepted":1}}}`)), StatusCode: 200} + + err := agent.Scrobble(ctx, "user-1", scrobbler.Scrobble{MediaFile: *track, TimeStamp: time.Now()}) + + Expect(err).ToNot(HaveOccurred()) + + body, _ := io.ReadAll(httpClient.SavedRequest.Body) + sentParams, _ := url.ParseQuery(string(body)) + Expect(sentParams.Has("duration")).To(BeTrue()) + Expect(sentParams.Get("duration")).To(Equal("30")) + }) + + It("skips songs according to the configured minimum duration", func() { + conf.Update(func(c *conf.Options) { c.LastFM.MinScrobbleDuration = 60 * time.Second }) + track.Duration = 59 + httpClient.Res = http.Response{Body: io.NopCloser(bytes.NewBufferString("{}")), StatusCode: 200} + + err := agent.Scrobble(ctx, "user-1", scrobbler.Scrobble{MediaFile: *track, TimeStamp: time.Now()}) + + Expect(err).ToNot(HaveOccurred()) + Expect(httpClient.SavedRequest).To(BeNil()) + }) + + It("does not skip short songs when the minimum duration is disabled", func() { + conf.Update(func(c *conf.Options) { c.LastFM.MinScrobbleDuration = 0 }) + track.Duration = 29 + httpClient.Res = http.Response{Body: io.NopCloser(bytes.NewBufferString("{}")), StatusCode: 200} + + err := agent.Scrobble(ctx, "user-1", scrobbler.Scrobble{MediaFile: *track, TimeStamp: time.Now()}) + + Expect(err).ToNot(HaveOccurred()) + Expect(httpClient.SavedRequest).ToNot(BeNil()) + + body, _ := io.ReadAll(httpClient.SavedRequest.Body) + sentParams, _ := url.ParseQuery(string(body)) + Expect(sentParams.Has("duration")).To(BeFalse()) + }) + It("returns ErrNotAuthorized if user is not linked", func() { err := agent.Scrobble(ctx, "user-2", scrobbler.Scrobble{MediaFile: *track, TimeStamp: time.Now()}) Expect(err).To(MatchError(scrobbler.ErrNotAuthorized)) diff --git a/adapters/lastfm/client.go b/adapters/lastfm/client.go index 7b2601cf9..286c54af7 100644 --- a/adapters/lastfm/client.go +++ b/adapters/lastfm/client.go @@ -182,7 +182,14 @@ func (c *client) scrobble(ctx context.Context, sessionKey string, info ScrobbleI params.Add("album", info.album) params.Add("trackNumber", strconv.Itoa(info.trackNumber)) params.Add("mbid", info.mbid) - params.Add("duration", strconv.Itoa(info.duration)) + + // Last.fm requires a minimum duration of 30 seconds to scrobble a track. To circumvent this, only send the duration if it is greater than or equal to 30 seconds. + if info.duration >= 30 { + params.Add("duration", strconv.Itoa(info.duration)) + } else { + log.Debug(ctx, "LastFM: scrobble duration is less than 30 seconds, duration will not be transmitted", "duration", info.duration) + } + params.Add("albumArtist", info.albumArtist) params.Add("sk", sessionKey) resp, err := c.makeRequest(ctx, http.MethodPost, params, true) diff --git a/conf/configuration.go b/conf/configuration.go index 676252c47..c0a23a005 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -176,6 +176,7 @@ type scannerOptions struct { GroupAlbumReleases bool // Deprecated: Use PID.Album instead FollowSymlinks bool // Whether to follow symlinks when scanning directories IgnoreDotFolders bool // Whether to ignore folders whose name starts with a dot when scanning + RespectNoMedia bool // Whether to skip folders (and their subfolders) containing a .nomedia file PurgeMissing string // Values: "never", "always", "full" } @@ -210,6 +211,7 @@ type lastfmOptions struct { Secret string //nolint:gosec Language string ScrobbleFirstArtistOnly bool + MinScrobbleDuration time.Duration // Computed values Languages []string `conf:"-"` // Computed from Language, split by comma @@ -424,6 +426,8 @@ func Load(noConfigDump bool) { durationNonNegativeOrDefault(&s.DevInsightsInitialDelay, consts.InsightsInitialDelay) durationNonNegativeOrDefault(&s.DevPluginCompilationTimeout, consts.DefaultPluginCompilationTimeout) + durationNonNegativeOrDefault(&s.LastFM.MinScrobbleDuration, consts.DefaultMinScrobbleDuration) + // Log deprecated, removed and unknown options for _, o := range deprecatedOptions { logDeprecatedOptions(o.name, o.replacement) @@ -1054,6 +1058,7 @@ func setViperDefaults() { viper.SetDefault("scanner.groupalbumreleases", false) viper.SetDefault("scanner.followsymlinks", true) viper.SetDefault("scanner.ignoredotfolders", true) + viper.SetDefault("scanner.respectnomedia", false) viper.SetDefault("scanner.purgemissing", consts.PurgeMissingNever) viper.SetDefault("subsonic.appendsubtitle", true) viper.SetDefault("subsonic.appendalbumversion", true) @@ -1071,6 +1076,7 @@ func setViperDefaults() { viper.SetDefault("lastfm.apikey", "") viper.SetDefault("lastfm.secret", "") viper.SetDefault("lastfm.scrobblefirstartistonly", false) + viper.SetDefault("lastfm.minscrobbleduration", consts.DefaultMinScrobbleDuration) viper.SetDefault("deezer.enabled", true) viper.SetDefault("deezer.language", consts.DefaultInfoLanguage) viper.SetDefault("listenbrainz.enabled", true) diff --git a/conf/configuration_test.go b/conf/configuration_test.go index 024949c6b..efd1554fb 100644 --- a/conf/configuration_test.go +++ b/conf/configuration_test.go @@ -404,6 +404,31 @@ var _ = Describe("Configuration", func() { }) }) + Describe("MinScrobbleDuration", func() { + It("defaults to 30 seconds", func() { + conf.Load(true) + Expect(conf.Server().LastFM.MinScrobbleDuration).To(Equal(consts.DefaultMinScrobbleDuration)) + }) + + It("sets negative values to default", func() { + viper.SetDefault("lastfm.minscrobbleduration", "-10s") + conf.Load(true) + Expect(conf.Server().LastFM.MinScrobbleDuration).To(Equal(consts.DefaultMinScrobbleDuration)) + }) + + It("disables with zero value", func() { + viper.SetDefault("lastfm.minscrobbleduration", "0s") + conf.Load(true) + Expect(conf.Server().LastFM.MinScrobbleDuration).To(Equal(time.Duration(0))) + }) + + It("preserves positive values", func() { + viper.SetDefault("lastfm.minscrobbleduration", "45s") + conf.Load(true) + Expect(conf.Server().LastFM.MinScrobbleDuration).To(Equal(45 * time.Second)) + }) + }) + Describe("Scanner.Extractor", func() { BeforeEach(func() { viper.Reset() diff --git a/consts/consts.go b/consts/consts.go index 42e9ec42f..9ed5c95bd 100644 --- a/consts/consts.go +++ b/consts/consts.go @@ -83,6 +83,8 @@ const ( DefaultInfoLanguage = "en" + DefaultMinScrobbleDuration = 30 * time.Second + ArtistInfoTimeToLive = 24 * time.Hour AlbumInfoTimeToLive = 7 * 24 * time.Hour UpdateLastAccessFrequency = time.Minute @@ -90,6 +92,7 @@ const ( I18nFolder = "i18n" ScanIgnoreFile = ".ndignore" + NoMediaFile = ".nomedia" ArtworkFolder = "artwork" // HashedArtworkFolder is a subtree of ArtworkFolder, kept apart from the name-addressed // upload folders beside it so Prune's sweep never reaches them. diff --git a/persistence/library_repository.go b/persistence/library_repository.go index 3d5363f74..b51e6bcac 100644 --- a/persistence/library_repository.go +++ b/persistence/library_repository.go @@ -136,10 +136,14 @@ ON CONFLICT (user_id, library_id) DO NOTHING;`, // TODO Remove this method when we have a proper UI to add libraries // This is a temporary method to store the music folder path from the config in the DB func (r *libraryRepository) StoreMusicFolder(ctx context.Context) error { + changed := And{Eq{"id": model.DefaultLibraryID}, NotEq{"path": conf.Server().MusicFolder}} + // An UPDATE takes the write lock even when nothing matches, and a startup scan may be holding it + if needsUpdate, err := r.exists(ctx, changed); err != nil || !needsUpdate { + return err + } sq := Update(r.tableName).Set("path", conf.Server().MusicFolder). Set("updated_at", time.Now()). - Where(Eq{"id": model.DefaultLibraryID}). - Where(NotEq{"path": conf.Server().MusicFolder}) + Where(changed) rowsAffected, err := r.executeSQL(ctx, sq) if err == nil && rowsAffected > 0 { libLock.Lock() diff --git a/persistence/library_repository_test.go b/persistence/library_repository_test.go index 0da217876..5c2e6bf6f 100644 --- a/persistence/library_repository_test.go +++ b/persistence/library_repository_test.go @@ -6,6 +6,7 @@ import ( "github.com/Masterminds/squirrel" "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/log" "github.com/navidrome/navidrome/model" "github.com/navidrome/navidrome/model/request" @@ -168,6 +169,18 @@ var _ = Describe("LibraryRepository", func() { Expect(libAfter.UpdatedAt).To(Equal(libBefore.UpdatedAt)) }) + It("does not need the write lock when the configured path is unchanged", func() { + conf.Update(func(c *conf.Options) { c.MusicFolder = libBefore.Path }) + lockConn, err := db.Db().Conn(ctx) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(lockConn.Close) + _, err = lockConn.ExecContext(ctx, "BEGIN IMMEDIATE") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _, _ = lockConn.ExecContext(context.Background(), "ROLLBACK") }) + + Expect(repo.StoreMusicFolder(ctx)).To(Succeed()) + }) + It("updates the default library only when the configured path changes", func() { conf.Update(func(c *conf.Options) { c.MusicFolder = libBefore.Path + "-updated" }) Expect(repo.StoreMusicFolder(ctx)).To(Succeed()) diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index 6369c5b71..324c88758 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -545,7 +545,18 @@ func (r *mediaFileRepository) FindRecentFilesByMBZTrackID(ctx context.Context, m // FindRecentFilesByProperties finds recently added files by intrinsic properties in other libraries // It uses a lightweight query without annotation/bookmark joins since those are not needed for matching func (r *mediaFileRepository) FindRecentFilesByProperties(ctx context.Context, missing model.MediaFile, since time.Time) (model.MediaFiles, error) { - sel := r.newSelect(ctx).Columns("media_file.*", "library.path as library_path", "library.name as library_name"). + var res dbMediaFiles + err := r.queryAll(ctx, recentFilesByPropertiesQuery(missing, since), &res) + if err != nil { + return nil, err + } + return res.toModels(), nil +} + +func recentFilesByPropertiesQuery(missing model.MediaFile, since time.Time) SelectBuilder { + // Many files share mbz_release_track_id = '', so the planner's pick of that index scans them all + return Select("media_file.*", "library.path as library_path", "library.name as library_name"). + From("media_file INDEXED BY media_file_title"). LeftJoin("library on media_file.library_id = library.id"). Where(And{ NotEq{"media_file.library_id": missing.LibraryID}, @@ -559,13 +570,6 @@ func (r *mediaFileRepository) FindRecentFilesByProperties(ctx context.Context, m Gt{"media_file.created_at": since}, Eq{"media_file.missing": false}, }).OrderBy("media_file.created_at DESC") - - var res dbMediaFiles - err := r.queryAll(ctx, sel, &res) - if err != nil { - return nil, err - } - return res.toModels(), nil } var mediaFileSearchConfig = searchConfig{ diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go index c782b44b4..69a9d318e 100644 --- a/persistence/mediafile_repository_test.go +++ b/persistence/mediafile_repository_test.go @@ -1203,6 +1203,73 @@ var _ = Describe("MediaRepository", func() { }) }) + Describe("FindRecentFilesByProperties", func() { + var adminCtx context.Context + var otherLib model.Library + var missing model.MediaFile + var since time.Time + + BeforeEach(func() { + adminCtx, otherLib, _ = restrictedFixture("moved") + since = time.Now().Add(-time.Hour) + missing = model.MediaFile{ + LibraryID: 1, Title: "Moved Song", Album: "Moved Album", Size: 1234, + Suffix: "flac", DiscNumber: 1, TrackNumber: 3, + } + }) + + candidate := func(id string, change func(*model.MediaFile)) { + mf := missing + mf.ID, mf.LibraryID, mf.Path, mf.CreatedAt = id, otherLib.ID, id+".flac", since.Add(time.Minute) + if change != nil { + change(&mf) + } + Expect(mr.Put(adminCtx, &mf)).To(Succeed()) + DeferCleanup(func() { _ = mr.Delete(adminCtx, mf.ID) }) + } + + It("returns only recent, present, untagged copies from other libraries", func() { + candidate("moved-match", nil) + candidate("moved-same-lib", func(mf *model.MediaFile) { mf.LibraryID = 1 }) + candidate("moved-tagged", func(mf *model.MediaFile) { mf.MbzReleaseTrackID = "some-mbid" }) + candidate("moved-older", func(mf *model.MediaFile) { mf.CreatedAt = since.Add(-time.Minute) }) + candidate("moved-missing", func(mf *model.MediaFile) { mf.Missing = true }) + candidate("moved-other-title", func(mf *model.MediaFile) { mf.Title = "Another Song" }) + + results, err := mr.FindRecentFilesByProperties(adminCtx, missing, since) + Expect(err).ToNot(HaveOccurred()) + Expect(slice.Map(results, func(mf model.MediaFile) string { return mf.ID })).To(ConsistOf("moved-match")) + Expect(results[0].LibraryPath).To(Equal(otherLib.Path)) + }) + + It("searches by title even when stats make the MBZ track ID index look more selective", func() { + conn := GetDBXBuilder().DB() + tx, err := conn.BeginTx(adminCtx, nil) + Expect(err).ToNot(HaveOccurred()) + // The schema, and so the loaded stats, are shared by every connection to the in-memory DB + DeferCleanup(func() { + Expect(tx.Rollback()).To(Succeed()) + _, err := conn.Exec("ANALYZE sqlite_schema") + Expect(err).ToNot(HaveOccurred()) + }) + for _, stmt := range []string{ + "ANALYZE media_file_title", + "DELETE FROM sqlite_stat1 WHERE tbl = 'media_file'", + "INSERT INTO sqlite_stat1 VALUES ('media_file', 'media_file_title', '40000 2'), ('media_file', 'media_file_mbz_release_track_id', '40000 1'), ('media_file', 'media_file_created_at', '40000 1')", + "ANALYZE sqlite_schema", + } { + _, err := tx.Exec(stmt) + Expect(err).ToNot(HaveOccurred(), stmt) + } + + query, args, err := recentFilesByPropertiesQuery(missing, since).ToSql() + Expect(err).ToNot(HaveOccurred()) + plan, err := queryPlan(tx, query, args...) + Expect(err).ToNot(HaveOccurred()) + Expect(plan).To(ContainElement(ContainSubstring("USING INDEX media_file_title"))) + }) + }) + Describe("dbMediaFile.String", func() { It("does not dereference a nil MediaFile", func() { Expect(fmt.Sprint(dbMediaFile{})).To(Equal("")) diff --git a/persistence/sort_index_coverage_test.go b/persistence/sort_index_coverage_test.go index eb4faf8a3..bf88ad713 100644 --- a/persistence/sort_index_coverage_test.go +++ b/persistence/sort_index_coverage_test.go @@ -140,24 +140,30 @@ var _ = Describe("Sort index coverage", func() { // "USE TEMP B-TREE FOR LAST TERM OF ORDER BY" (or "LAST N TERMS") and are allowed. var fullTempBTreeSort = regexp.MustCompile(`USE TEMP B-TREE FOR ORDER BY`) -func checkSortUsesIndex(conn *sql.DB, table, orderBy string) error { - rows, err := conn.Query(fmt.Sprintf("explain query plan select * from %s order by %s limit 15", table, orderBy)) +func queryPlan(conn interface { + Query(string, ...any) (*sql.Rows, error) +}, query string, args ...any) ([]string, error) { + rows, err := conn.Query("explain query plan "+query, args...) if err != nil { - return fmt.Errorf("explain query plan failed for order by %q: %w", orderBy, err) + return nil, err } defer rows.Close() var details []string for rows.Next() { - var id, parent, notUsed int var detail string - if err := rows.Scan(&id, &parent, ¬Used, &detail); err != nil { - return err + if err := rows.Scan(new(int), new(int), new(int), &detail); err != nil { + return nil, err } details = append(details, detail) } - if err := rows.Err(); err != nil { - return err + return details, rows.Err() +} + +func checkSortUsesIndex(conn *sql.DB, table, orderBy string) error { + details, err := queryPlan(conn, fmt.Sprintf("select * from %s order by %s limit 15", table, orderBy)) + if err != nil { + return fmt.Errorf("explain query plan failed for order by %q: %w", orderBy, err) } if slices.ContainsFunc(details, fullTempBTreeSort.MatchString) { return fmt.Errorf("no index satisfies ORDER BY %s - plan: %v", orderBy, details) diff --git a/scanner/README.md b/scanner/README.md index 7d921b2b9..6245a0d34 100644 --- a/scanner/README.md +++ b/scanner/README.md @@ -175,7 +175,7 @@ flowchart TD 1. **Folder Traversal** - Uses `walkDirTree` to traverse the directory structure - Handles symbolic links and hidden files - - Processes `.ndignore` files for exclusions + - Processes `.ndignore` files for exclusions, and skips folders containing a `.nomedia` file when `Scanner.RespectNoMedia` is enabled - Maps files to appropriate types (audio, image, playlist) 2. **Metadata Extraction** diff --git a/scanner/ignore_checker.go b/scanner/ignore_checker.go index f0aedb079..307170ea6 100644 --- a/scanner/ignore_checker.go +++ b/scanner/ignore_checker.go @@ -7,11 +7,14 @@ import ( "path" "strings" + "github.com/navidrome/navidrome/conf" "github.com/navidrome/navidrome/consts" "github.com/navidrome/navidrome/log" ignore "github.com/sabhiram/go-gitignore" ) +const ignoreAllPattern = "**/*" + // IgnoreChecker manages .ndignore patterns using a stack-based approach. // Use Push() to add patterns when entering a folder, Pop() when leaving, // and ShouldIgnore() to check if a path should be ignored. @@ -80,6 +83,24 @@ func (ic *IgnoreChecker) PushAllParents(ctx context.Context, targetPath string) return nil } +// PushParents pushes the patterns of each folder above target, from the root down. It returns true, +// and stops, when target or a folder above it is ignored, as a full walk would never reach target. +func (ic *IgnoreChecker) PushParents(ctx context.Context, target string) bool { + target = path.Clean(target) + if target == "." { + return false + } + dir := "." + for part := range strings.SplitSeq(target, "/") { + _ = ic.Push(ctx, dir) + dir = path.Join(dir, part) + if ic.ShouldIgnore(ctx, dir) { + return true + } + } + return false +} + // ShouldIgnore checks if the given path should be ignored based on the current patterns. // Returns true if the path matches any ignore pattern, false otherwise. func (ic *IgnoreChecker) ShouldIgnore(ctx context.Context, relPath string) bool { @@ -95,15 +116,22 @@ func (ic *IgnoreChecker) ShouldIgnore(ctx context.Context, relPath string) bool matches := ic.matcher.MatchesPath(relPath) if matches { - log.Trace(ctx, "Scanner: Ignoring entry matching .ndignore", "path", relPath) + log.Trace(ctx, "Scanner: Ignoring entry matching ignore rules", "path", relPath) } return matches } // loadPatternsFromFolder reads the .ndignore file in the specified folder and returns the patterns. // If the file doesn't exist, returns an empty slice. -// If the file exists but is empty, returns a pattern to ignore everything ("**/*"). +// If the file exists but is empty, or there is a .nomedia file with Scanner.RespectNoMedia, ignores everything. func (ic *IgnoreChecker) loadPatternsFromFolder(ctx context.Context, folder string) []string { + if conf.Server().Scanner.RespectNoMedia { + if _, err := fs.Stat(ic.fsys, path.Join(folder, consts.NoMediaFile)); err == nil { + log.Trace(ctx, "Scanner: .nomedia file found, ignoring everything", "path", folder) + return []string{ignoreAllPattern} + } + } + ignoreFilePath := path.Join(folder, consts.ScanIgnoreFile) var patterns []string @@ -138,7 +166,7 @@ func (ic *IgnoreChecker) loadPatternsFromFolder(ctx context.Context, folder stri // If the .ndignore file is empty, ignore everything if len(patterns) == 0 { log.Trace(ctx, "Scanner: .ndignore file is empty, ignoring everything", "path", folder) - patterns = []string{"**/*"} + patterns = []string{ignoreAllPattern} } return patterns diff --git a/scanner/ignore_checker_test.go b/scanner/ignore_checker_test.go index 5378ed4fa..69c634440 100644 --- a/scanner/ignore_checker_test.go +++ b/scanner/ignore_checker_test.go @@ -4,6 +4,7 @@ import ( "context" "testing/fstest" + "github.com/navidrome/navidrome/conf" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -37,6 +38,26 @@ var _ = Describe("IgnoreChecker", func() { }) }) + DescribeTable("when .nomedia file exists", + func(respectNoMedia bool, files fstest.MapFS, expected []string) { + conf.Update(func(c *conf.Options) { c.Scanner.RespectNoMedia = respectNoMedia }) + ic = newIgnoreChecker(files) + patterns := ic.loadPatternsFromFolder(ctx, ".") + Expect(patterns).To(Equal(expected)) + }, + Entry("empty", true, fstest.MapFS{".nomedia": &fstest.MapFile{}}, []string{"**/*"}), + Entry("with content", true, fstest.MapFS{".nomedia": &fstest.MapFile{Data: []byte("*.txt")}}, []string{"**/*"}), + Entry("alongside .ndignore", true, fstest.MapFS{ + ".nomedia": &fstest.MapFile{}, + ".ndignore": &fstest.MapFile{Data: []byte("*.txt")}, + }, []string{"**/*"}), + Entry("disabled", false, fstest.MapFS{".nomedia": &fstest.MapFile{}}, []string(nil)), + Entry("disabled, alongside .ndignore", false, fstest.MapFS{ + ".nomedia": &fstest.MapFile{}, + ".ndignore": &fstest.MapFile{Data: []byte("*.txt")}, + }, []string{"*.txt"}), + ) + DescribeTable("parsing .ndignore content", func(content string, expectedPatterns []string) { fsys := fstest.MapFS{ diff --git a/scanner/scanner_internal_test.go b/scanner/scanner_internal_test.go index e8abb7c7d..ab39da0f0 100644 --- a/scanner/scanner_internal_test.go +++ b/scanner/scanner_internal_test.go @@ -44,7 +44,8 @@ var _ = Describe("runPhase", func() { ) BeforeEach(func() { - ctx = context.Background() + ctx = GinkgoT().Context() + sum.Store(0) phaseNum = 1 phase = &mockPhase{ num: 3, diff --git a/scanner/scanner_multilibrary_test.go b/scanner/scanner_multilibrary_test.go index 0d439879d..87782b4a0 100644 --- a/scanner/scanner_multilibrary_test.go +++ b/scanner/scanner_multilibrary_test.go @@ -3,6 +3,7 @@ package scanner_test import ( "context" "errors" + "os" "path/filepath" "testing/fstest" "time" @@ -41,9 +42,13 @@ var _ = Describe("Scanner - Multi-Library", Ordered, func() { } BeforeAll(func() { - tests.SkipOnWindows("SQLite file lock blocks TempDir cleanup (#TBD-path-sep-scanner)") ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true}) - tmpDir := GinkgoT().TempDir() + // db.Db() is a process-wide singleton, so this dir may hold the open DB until + // the suite ends, and Windows can't delete an open file + tmpDir, err := os.MkdirTemp("", "scanner-test") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = os.RemoveAll(tmpDir) }) + DeferCleanup(conf.SnapshotConfig()) conf.Update(func(c *conf.Options) { c.DbPath = filepath.Join(tmpDir, "test-scanner-multilibrary.db?_journal_mode=WAL") }) diff --git a/scanner/scanner_selective_test.go b/scanner/scanner_selective_test.go index b3edd6f1a..a9c30260e 100644 --- a/scanner/scanner_selective_test.go +++ b/scanner/scanner_selective_test.go @@ -2,6 +2,7 @@ package scanner_test import ( "context" + "os" "path/filepath" "testing/fstest" "time" @@ -34,9 +35,13 @@ var _ = Describe("ScanFolders", Ordered, func() { var fsys storagetest.FakeFS BeforeAll(func() { - tests.SkipOnWindows("SQLite file lock blocks TempDir cleanup (#TBD-path-sep-scanner)") ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true}) - tmpDir := GinkgoT().TempDir() + // db.Db() is a process-wide singleton, so this dir may hold the open DB until + // the suite ends, and Windows can't delete an open file + tmpDir, err := os.MkdirTemp("", "scanner-test") + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(func() { _ = os.RemoveAll(tmpDir) }) + DeferCleanup(conf.SnapshotConfig()) conf.Update(func(c *conf.Options) { c.DbPath = filepath.Join(tmpDir, "test-selective-scan.db?_journal_mode=WAL") }) log.Warn("Using DB at " + conf.Server().DbPath) db.Db().SetMaxOpenConns(1) diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go index 30dc23c4f..e28233e77 100644 --- a/scanner/scanner_test.go +++ b/scanner/scanner_test.go @@ -59,6 +59,7 @@ var _ = Describe("Scanner", Ordered, func() { tmpDir, err := os.MkdirTemp("", "scanner-test") Expect(err).ToNot(HaveOccurred()) DeferCleanup(func() { _ = os.RemoveAll(tmpDir) }) + DeferCleanup(conf.SnapshotConfig()) conf.Update(func(c *conf.Options) { c.DbPath = filepath.Join(tmpDir, "test-scanner.db?_journal_mode=WAL") }) log.Warn("Using DB at " + conf.Server().DbPath) //conf.Server.DbPath = ":memory:" diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index 4ab28d1d2..fb45637da 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -50,11 +50,10 @@ func walkDirTree(ctx context.Context, job *scanJob, targetFolders ...string) (<- continue } - // Create checker and push patterns from root to this folder + // walkFolder pushes the target folder's own patterns, so only its parents are pushed here checker := newIgnoreChecker(job.fs) - err = checker.PushAllParents(ctx, folderPath) - if err != nil { - log.Error(ctx, "Scanner: Error pushing ignore patterns for target folder", "path", folderPath, err) + if checker.PushParents(ctx, folderPath) { + log.Debug(ctx, "Scanner: Skipping target folder ignored by its parents", "path", folderPath) continue } diff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.go index 6f1d52bcc..33015db35 100644 --- a/scanner/walk_dir_tree_test.go +++ b/scanner/walk_dir_tree_test.go @@ -27,8 +27,26 @@ var _ = Describe("walk_dir_tree", func() { ctx context.Context ) + // Helper function to call walkDirTree and collect folders from the results channel + getFolders := func(targets ...string) map[string]*folderEntry { + results, err := walkDirTree(ctx, job, targets...) + Expect(err).ToNot(HaveOccurred()) + + folders := map[string]*folderEntry{} + g := errgroup.Group{} + g.Go(func() error { + for folder := range results { + folders[folder.path] = folder + } + return nil + }) + _ = g.Wait() + return folders + } + Context("full library", func() { BeforeEach(func() { + conf.Update(func(c *conf.Options) { c.Scanner.RespectNoMedia = true }) ctx = GinkgoT().Context() fsys = &mockMusicFS{ FS: fstest.MapFS{ @@ -53,6 +71,9 @@ var _ = Describe("walk_dir_tree", func() { "root/h/.hidden.mp3": {}, "root/i/.git/config": {}, "root/i/.streams/stream.mp3": {}, + "root/j/.nomedia": {}, + "root/j/f1.mp3": {}, + "root/j/sub/f2.mp3": {}, }, } job = &scanJob{ @@ -61,23 +82,6 @@ var _ = Describe("walk_dir_tree", func() { } }) - // Helper function to call walkDirTree and collect folders from the results channel - getFolders := func() map[string]*folderEntry { - results, err := walkDirTree(ctx, job) - Expect(err).ToNot(HaveOccurred()) - - folders := map[string]*folderEntry{} - g := errgroup.Group{} - g.Go(func() error { - for folder := range results { - folders[folder.path] = folder - } - return nil - }) - _ = g.Wait() - return folders - } - DescribeTable("symlink handling", func(followSymlinks bool, expectedFolderCount int) { conf.Update(func(c *conf.Options) { c.Scanner.FollowSymlinks = followSymlinks }) @@ -100,6 +104,8 @@ var _ = Describe("walk_dir_tree", func() { Expect(folders["root/c"].audioFiles).To(BeEmpty()) Expect(folders["root/c"].imageFiles).To(BeEmpty()) Expect(folders).ToNot(HaveKey("root/d")) + Expect(folders["root/j"].audioFiles).To(BeEmpty()) + Expect(folders).ToNot(HaveKey("root/j/sub")) // By default (Scanner.IgnoreDotFolders == true), dot-prefixed // folders are skipped, dot-prefixed files are not indexed, and @@ -122,8 +128,8 @@ var _ = Describe("walk_dir_tree", func() { Expect(folders["root/f"].audioFiles).ToNot(HaveKey("evil.mp3")) } }, - Entry("with symlinks enabled", true, 11), - Entry("with symlinks disabled", false, 10), + Entry("with symlinks enabled", true, 12), + Entry("with symlinks disabled", false, 11), ) DescribeTable("dot-prefixed folders with IgnoreDotFolders disabled", @@ -150,6 +156,37 @@ var _ = Describe("walk_dir_tree", func() { Entry("with symlinks enabled", true), Entry("with symlinks disabled", false), ) + + It("scans folders containing a .nomedia file when RespectNoMedia is disabled", func() { + conf.Update(func(c *conf.Options) { c.Scanner.RespectNoMedia = false }) + folders := getFolders() + + Expect(folders["root/j"].audioFiles).To(HaveKey("f1.mp3")) + Expect(folders["root/j/sub"].audioFiles).To(HaveKey("f2.mp3")) + }) + }) + + Context("targeted scans", func() { + BeforeEach(func() { + conf.Update(func(c *conf.Options) { c.Scanner.RespectNoMedia = true }) + ctx = GinkgoT().Context() + }) + + DescribeTable("skips a folder ignored by its parent, even if it re-includes files", + func(targets []string) { + job = &scanJob{fs: &mockMusicFS{FS: fstest.MapFS{ + "a/.nomedia": {}, + "a/b/.ndignore": {Data: []byte("!song.mp3\n!c")}, + "a/b/song.mp3": {}, + "a/b/c/song.mp3": {}, + }}, lib: model.Library{Path: "/music"}} + + Expect(getFolders(targets...)).ToNot(SatisfyAny(HaveKey("a/b"), HaveKey("a/b/c"))) + }, + Entry("full scan", nil), + Entry("targeted scan", []string{"a/b"}), + Entry("targeted scan of a deeper folder", []string{"a/b/c"}), + ) }) Context("with target folders", func() { diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go index fa6b26980..1a08e263e 100644 --- a/scanner/watcher_test.go +++ b/scanner/watcher_test.go @@ -432,6 +432,18 @@ var _ = Describe("Watcher", func() { }) }) + DescribeTable("with a .nomedia file in a parent folder", + func(respectNoMedia bool, expected bool) { + conf.Update(func(c *conf.Options) { c.Scanner.RespectNoMedia = respectNoMedia }) + fsys := &mockMusicFS{FS: fstest.MapFS{ + "rock/.nomedia": &fstest.MapFile{}, + "rock/album/song.mp3": &fstest.MapFile{}, + }} + Expect(w.shouldIgnoreFolderPath(ctx, fsys, "rock/album")).To(Equal(expected)) + }, + Entry("ignores the folder when RespectNoMedia is enabled", true, true), + Entry("scans the folder when RespectNoMedia is disabled", false, false), + ) }) }) diff --git a/server/initial_setup.go b/server/initial_setup.go index 8c48c6f44..06f69b8db 100644 --- a/server/initial_setup.go +++ b/server/initial_setup.go @@ -14,31 +14,23 @@ import ( "github.com/navidrome/navidrome/model/id" ) -func initialSetup(ds model.DataStore) { - ctx := context.TODO() - err := ds.WithTx(func(tx model.DataStore) error { - if err := tx.Library().StoreMusicFolder(ctx); err != nil { - return err - } - - properties := tx.Property() - _, err := properties.Get(ctx, consts.InitialSetupFlagKey) - if err == nil { - return nil - } +// Keeps reads out of the transaction: SQLite fails a write that follows a read at once, without waiting for the lock +func initialSetup(ctx context.Context, ds model.DataStore) error { + if err := ds.Library().StoreMusicFolder(ctx); err != nil { + return err + } + if _, err := ds.Property().Get(ctx, consts.InitialSetupFlagKey); err == nil { + return nil + } + return ds.WithTxRetry(ctx, func(ctx context.Context, tx model.DataStore) error { log.Info("Running initial setup") if conf.Server().DevAutoCreateAdminPassword != "" { - if err = createInitialAdminUser(ctx, tx, conf.Server().DevAutoCreateAdminPassword); err != nil { + if err := createInitialAdminUser(ctx, tx, conf.Server().DevAutoCreateAdminPassword); err != nil { return err } } - - err = properties.Put(ctx, consts.InitialSetupFlagKey, time.Now().String()) - return err + return tx.Property().Put(ctx, consts.InitialSetupFlagKey, time.Now().String()) }, "initial setup") - if err != nil { - log.Fatal("Error running initial setup", err) - } } // If the Dev Admin user is not present, create it diff --git a/server/initial_setup_test.go b/server/initial_setup_test.go index 0c85d9d0a..1106f4c95 100644 --- a/server/initial_setup_test.go +++ b/server/initial_setup_test.go @@ -3,8 +3,14 @@ package server import ( "context" "errors" + "path/filepath" + "time" + "github.com/navidrome/navidrome/conf" + "github.com/navidrome/navidrome/consts" + "github.com/navidrome/navidrome/db" "github.com/navidrome/navidrome/model" + "github.com/navidrome/navidrome/persistence" "github.com/navidrome/navidrome/tests" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -59,3 +65,61 @@ var _ = Describe("initial_setup", func() { }) }) }) + +// db.Db() is a process-wide singleton that the BeforeAll cleanup closes; keep these the only real-DB specs in this package. +var _ = Describe("initialSetup with a real database", Ordered, func() { + var ctx context.Context + var ds model.DataStore + + holdWriteLock := func(releaseAfter time.Duration) { + conn, err := db.Db().Conn(ctx) + Expect(err).ToNot(HaveOccurred()) + _, err = conn.ExecContext(ctx, "BEGIN IMMEDIATE") + Expect(err).ToNot(HaveOccurred()) + released := make(chan struct{}) + go func() { + defer close(released) + time.Sleep(releaseAfter) + _, _ = conn.ExecContext(context.Background(), "ROLLBACK") + _ = conn.Close() + }() + DeferCleanup(func() { <-released }) + } + + BeforeAll(func() { + DeferCleanup(conf.SnapshotConfig()) + conf.Update(func(c *conf.Options) { + c.DbPath = filepath.Join(GinkgoT().TempDir(), "initial-setup.db") + "?_journal_mode=WAL&_busy_timeout=1000" + }) + DeferCleanup(db.Init(GinkgoT().Context())) + ds = persistence.New(db.Db()) + }) + + BeforeEach(func() { + ctx = GinkgoT().Context() + }) + + It("completes the first-run setup when the write lock is held briefly", func() { + holdWriteLock(300 * time.Millisecond) + + Expect(initialSetup(ctx, ds)).To(Succeed()) + Expect(ds.Property().Get(ctx, consts.InitialSetupFlagKey)).ToNot(BeEmpty()) + }) + + It("does not need the write lock when setup already ran and the music folder is unchanged", func() { + holdWriteLock(1500 * time.Millisecond) + + Expect(initialSetup(ctx, ds)).To(Succeed()) + }) + + It("stores a changed music folder when the write lock is held briefly", func() { + newFolder := conf.Server().MusicFolder + "-moved" + conf.Update(func(c *conf.Options) { c.MusicFolder = newFolder }) + holdWriteLock(300 * time.Millisecond) + + Expect(initialSetup(ctx, ds)).To(Succeed()) + lib, err := ds.Library().Get(ctx, model.DefaultLibraryID) + Expect(err).ToNot(HaveOccurred()) + Expect(lib.Path).To(Equal(newFolder)) + }) +}) diff --git a/server/server.go b/server/server.go index e9b6d20cf..5ad61e3e9 100644 --- a/server/server.go +++ b/server/server.go @@ -37,7 +37,9 @@ type Server struct { func New(ds model.DataStore, broker events.Broker, insights metrics.Insights) *Server { s := &Server{ds: ds, broker: broker, insights: insights} - initialSetup(ds) + if err := initialSetup(context.TODO(), ds); err != nil { + log.Fatal("Error running initial setup", err) + } auth.Init(s.ds) s.initRoutes() s.mountAuthenticationRoutes() diff --git a/ui/src/layout/Login.jsx b/ui/src/layout/Login.jsx index a7763cff3..9e79d3c45 100644 --- a/ui/src/layout/Login.jsx +++ b/ui/src/layout/Login.jsx @@ -1,7 +1,8 @@ -import React, { useState, useCallback } from 'react' +import React, { useState, useCallback, useEffect } from 'react' import PropTypes from 'prop-types' import { Field, Form } from 'react-final-form' import { useDispatch } from 'react-redux' +import { useHistory } from 'react-router-dom' import Button from '@material-ui/core/Button' import Card from '@material-ui/core/Card' import CardActions from '@material-ui/core/CardActions' @@ -11,6 +12,7 @@ import TextField from '@material-ui/core/TextField' import { ThemeProvider, makeStyles } from '@material-ui/core/styles' import { createMuiTheme, + useCheckAuth, useLogin, useNotify, useTranslate, @@ -322,6 +324,16 @@ const Login = ({ location }) => { const notify = useNotify() const login = useLogin() const dispatch = useDispatch() + const checkAuth = useCheckAuth() + const history = useHistory() + + // Signed-in users opening #/login go straight to the app; + // replace keeps Back from bouncing. + useEffect(() => { + checkAuth({}, false) + .then(() => history.replace('/')) + .catch(() => {}) + }, [checkAuth, history]) const handleSubmit = useCallback( (auth) => { diff --git a/ui/src/layout/Login.test.jsx b/ui/src/layout/Login.test.jsx new file mode 100644 index 000000000..d2e2e5fc2 --- /dev/null +++ b/ui/src/layout/Login.test.jsx @@ -0,0 +1,70 @@ +import React from 'react' +import { render, screen, waitFor } from '@testing-library/react' +import { describe, it, beforeEach, vi } from 'vitest' +import { Provider } from 'react-redux' +import { createStore } from 'redux' +import { createMuiTheme } from '@material-ui/core/styles' +import Login from './Login' + +const mocks = vi.hoisted(() => ({ + checkAuth: vi.fn(), + push: vi.fn(), + replace: vi.fn(), +})) + +vi.mock('react-admin', () => ({ + createMuiTheme: (theme) => theme, + useCheckAuth: () => mocks.checkAuth, + useLogin: () => vi.fn(() => Promise.resolve()), + useNotify: () => vi.fn(), + useTranslate: () => (x) => x, + useVersion: () => 1, +})) + +vi.mock('react-router-dom', () => ({ + useHistory: () => ({ push: mocks.push, replace: mocks.replace }), +})) + +vi.mock('../themes/useCurrentTheme', () => ({ + default: () => createMuiTheme({}), +})) + +vi.mock('./Notification', () => ({ + default: () =>
, +})) + +const renderLogin = () => + render( + ({}))}> + + , + ) + +describe('', () => { + beforeEach(() => { + mocks.checkAuth.mockReset() + mocks.push.mockReset() + mocks.replace.mockReset() + }) + + it('replaces the login page with the home page when already signed in', async () => { + mocks.checkAuth.mockResolvedValue() + + renderLogin() + + await waitFor(() => expect(mocks.replace).toHaveBeenCalledWith('/')) + expect(mocks.checkAuth).toHaveBeenCalledWith({}, false) + expect(mocks.push).not.toHaveBeenCalled() + }) + + it('shows the login form when the user is not signed in', async () => { + mocks.checkAuth.mockRejectedValue() + + renderLogin() + + await waitFor(() => expect(mocks.checkAuth).toHaveBeenCalledWith({}, false)) + expect(screen.getByText('ra.auth.sign_in')).toBeInTheDocument() + expect(mocks.replace).not.toHaveBeenCalled() + expect(mocks.push).not.toHaveBeenCalled() + }) +})