diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go index b80ede082..77db41828 100644 --- a/scanner/walk_dir_tree.go +++ b/scanner/walk_dir_tree.go @@ -66,11 +66,14 @@ func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) { } stats.ModTime = dirInfo.ModTime() - dirEntries, err := fullReadDir(dirPath) + dir, err := os.Open(dirPath) if err != nil { - log.Error(ctx, "Error in ReadDir", "path", dirPath, err) + log.Error(ctx, "Error in Opening directory", "path", dirPath, err) return children, stats, err } + defer dir.Close() + + dirEntries := fullReadDir(ctx, dir) for _, entry := range dirEntries { isDir, err := isDirOrSymlinkToDir(dirPath, entry) // Skip invalid symlinks @@ -100,20 +103,28 @@ func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) { return children, stats, nil } -func fullReadDir(name string) ([]os.DirEntry, error) { - f, err := os.Open(name) - if err != nil { - return nil, err +// fullReadDir reads all files in the folder, skipping the ones with errors. +// It also detects when it is "stuck" with an error in the same directory over and over. +// In this case, it and returns whatever it was able to read until it got stuck. +// See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850 +func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry { + var allDirs []os.DirEntry + var prevErrStr = "" + for { + dirs, err := dir.ReadDir(-1) + allDirs = append(allDirs, dirs...) + if err == nil { + break + } + log.Warn(ctx, "Skipping DirEntry", err) + if prevErrStr == err.Error() { + log.Error(ctx, "Duplicate DirEntry failure, bailing", err) + break + } + prevErrStr = err.Error() } - defer f.Close() - - dirs, err := f.ReadDir(-1) - if err != nil { - log.Warn("Skipping DirEntry", err) - return nil, nil - } - sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() }) - return dirs, nil + sort.Slice(allDirs, func(i, j int) bool { return allDirs[i].Name() < allDirs[j].Name() }) + return allDirs } // isDirOrSymlinkToDir returns true if and only if the dirEnt represents a file diff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.go index 6c8464bfb..95a586aaa 100644 --- a/scanner/walk_dir_tree_test.go +++ b/scanner/walk_dir_tree_test.go @@ -2,8 +2,10 @@ package scanner import ( "context" + "io/fs" "os" "path/filepath" + "testing/fstest" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -78,8 +80,85 @@ var _ = Describe("walk_dir_tree", func() { Expect(isDirIgnored(baseDir, dirEntry)).To(BeFalse()) }) }) + + Describe("fullReadDir", func() { + var fsys fakeFS + var ctx context.Context + BeforeEach(func() { + ctx = context.Background() + fsys = fakeFS{MapFS: fstest.MapFS{ + "root/a/f1": {}, + "root/b/f2": {}, + "root/c/f3": {}, + }} + }) + It("reads all entries", func() { + dir, _ := fsys.Open("root") + entries := fullReadDir(ctx, dir.(fs.ReadDirFile)) + Expect(entries).To(HaveLen(3)) + Expect(entries[0].Name()).To(Equal("a")) + Expect(entries[1].Name()).To(Equal("b")) + Expect(entries[2].Name()).To(Equal("c")) + }) + It("skips entries with permission error", func() { + fsys.failOn = "b" + dir, _ := fsys.Open("root") + entries := fullReadDir(ctx, dir.(fs.ReadDirFile)) + Expect(entries).To(HaveLen(2)) + Expect(entries[0].Name()).To(Equal("a")) + Expect(entries[1].Name()).To(Equal("c")) + }) + It("aborts if it keeps getting 'readdirent: no such file or directory'", func() { + fsys.err = fs.ErrNotExist + dir, _ := fsys.Open("root") + entries := fullReadDir(ctx, dir.(fs.ReadDirFile)) + Expect(entries).To(BeEmpty()) + }) + }) }) +type fakeFS struct { + fstest.MapFS + failOn string + err error +} + +func (f *fakeFS) Open(name string) (fs.File, error) { + dir, err := f.MapFS.Open(name) + return &fakeDirFile{File: dir, fail: f.failOn, err: f.err}, err +} + +type fakeDirFile struct { + fs.File + entries []fs.DirEntry + pos int + fail string + err error +} + +// Only works with n == -1 +func (fd *fakeDirFile) ReadDir(n int) ([]fs.DirEntry, error) { + if fd.err != nil { + return nil, fd.err + } + if fd.entries == nil { + fd.entries, _ = fd.File.(fs.ReadDirFile).ReadDir(-1) + } + var dirs []fs.DirEntry + for { + if fd.pos >= len(fd.entries) { + break + } + e := fd.entries[fd.pos] + fd.pos++ + if e.Name() == fd.fail { + return dirs, &fs.PathError{Op: "lstat", Path: e.Name(), Err: fs.ErrPermission} + } + dirs = append(dirs, e) + } + return dirs, nil +} + func getDirEntry(baseDir, name string) (os.DirEntry, error) { dirEntries, _ := os.ReadDir(baseDir) for _, entry := range dirEntries {