From 52a4721c91dc3bf1eee05e1f96476455c4a63237 Mon Sep 17 00:00:00 2001 From: Deluan Date: Sat, 24 Dec 2022 18:53:09 -0500 Subject: [PATCH] Remove empty (invalid) entries from the cache --- utils/cache/file_caches.go | 2 +- utils/cache/file_haunter.go | 121 ++++++++++++++++++++++++++++++++ utils/cache/file_hauter_test.go | 81 +++++++++++++++++++++ 3 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 utils/cache/file_haunter.go create mode 100644 utils/cache/file_hauter_test.go diff --git a/utils/cache/file_caches.go b/utils/cache/file_caches.go index 00680ebe5..633f7bd10 100644 --- a/utils/cache/file_caches.go +++ b/utils/cache/file_caches.go @@ -210,7 +210,7 @@ func newFSCache(name, cacheSize, cacheFolder string, maxItems int) (fscache.Cach return nil, nil } - lru := fscache.NewLRUHaunter(maxItems, int64(size), consts.DefaultCacheCleanUpInterval) + lru := NewFileHaunter(maxItems, int64(size), consts.DefaultCacheCleanUpInterval) h := fscache.NewLRUHaunterStrategy(lru) cacheFolder = filepath.Join(conf.Server.DataFolder, cacheFolder) diff --git a/utils/cache/file_haunter.go b/utils/cache/file_haunter.go new file mode 100644 index 000000000..880332751 --- /dev/null +++ b/utils/cache/file_haunter.go @@ -0,0 +1,121 @@ +package cache + +import ( + "sort" + "time" + + "github.com/djherbis/fscache" + "github.com/navidrome/navidrome/log" +) + +type haunterKV struct { + key string + value fscache.Entry + info fscache.FileInfo +} + +// NewFileHaunter returns a simple haunter which runs every "period" +// and scrubs older files when the total file size is over maxSize or +// total item count is over maxItems. It also removes empty (invalid) files. +// If maxItems or maxSize are 0, they won't be checked +// +// Based on fscache.NewLRUHaunter +func NewFileHaunter(maxItems int, maxSize int64, period time.Duration) fscache.LRUHaunter { + return &fileHaunter{ + period: period, + maxItems: maxItems, + maxSize: maxSize, + } +} + +type fileHaunter struct { + period time.Duration + maxItems int + maxSize int64 +} + +func (j *fileHaunter) Next() time.Duration { + return j.period +} + +func (j *fileHaunter) Scrub(c fscache.CacheAccessor) (keysToReap []string) { + var count int + var size int64 + var okFiles []haunterKV + + c.EnumerateEntries(func(key string, e fscache.Entry) bool { + if e.InUse() { + return true + } + + fileInfo, err := c.Stat(e.Name()) + if err != nil { + return true + } + + if fileInfo.Size() == 0 { + log.Trace("Removing invalid empty file", "file", e.Name()) + keysToReap = append(keysToReap, key) + } + + count++ + size = size + fileInfo.Size() + okFiles = append(okFiles, haunterKV{ + key: key, + value: e, + info: fileInfo, + }) + + return true + }) + + sort.Slice(okFiles, func(i, j int) bool { + iLastRead := okFiles[i].info.AccessTime() + jLastRead := okFiles[j].info.AccessTime() + + return iLastRead.Before(jLastRead) + }) + + collectKeysToReapFn := func() bool { + var key *string + var err error + key, count, size, err = j.removeFirst(&okFiles, count, size) + if err != nil { + return false + } + if key != nil { + keysToReap = append(keysToReap, *key) + } + + return true + } + + if j.maxItems > 0 { + for count > j.maxItems { + if !collectKeysToReapFn() { + break + } + } + } + + if j.maxSize > 0 { + for size > j.maxSize { + if !collectKeysToReapFn() { + break + } + } + } + + return keysToReap +} + +func (j *fileHaunter) removeFirst(items *[]haunterKV, count int, size int64) (*string, int, int64, error) { + var f haunterKV + + f, *items = (*items)[0], (*items)[1:] + + count-- + size = size - f.info.Size() + + return &f.key, count, size, nil +} diff --git a/utils/cache/file_hauter_test.go b/utils/cache/file_hauter_test.go new file mode 100644 index 000000000..7b35ee47b --- /dev/null +++ b/utils/cache/file_hauter_test.go @@ -0,0 +1,81 @@ +package cache_test + +import ( + "fmt" + "io" + "os" + "path/filepath" + "testing" + "time" + + "github.com/djherbis/fscache" + "github.com/navidrome/navidrome/utils/cache" +) + +func TestFileHaunterMaxSize(t *testing.T) { + tempDir, _ := os.MkdirTemp("", "spread_fs") + cacheDir := filepath.Join(tempDir, "cache1") + fs, err := fscache.NewFs(cacheDir, 0700) + if err != nil { + t.Error(err.Error()) + t.FailNow() + } + defer os.RemoveAll(tempDir) + + c, err := fscache.NewCacheWithHaunter(fs, fscache.NewLRUHaunterStrategy(cache.NewFileHaunter(0, 24, 400*time.Millisecond))) + if err != nil { + t.Error(err.Error()) + return + } + defer c.Clean() //nolint:errcheck + + // Create 5 normal files and 1 empty + for i := 0; i < 6; i++ { + name := fmt.Sprintf("stream-%v", i) + var r fscache.ReadAtCloser + if i < 5 { + r = createCachedStream(c, name, "hello") + } else { // Last one is empty + r = createCachedStream(c, name, "") + } + + if !c.Exists(name) { + t.Errorf(name + " should exist") + } + + <-time.After(10 * time.Millisecond) + + err := r.Close() + if err != nil { + t.Error(err) + } + } + + <-time.After(400 * time.Millisecond) + + if c.Exists("stream-0") { + t.Errorf("stream-0 should have been scrubbed") + } + + if c.Exists("stream-5") { + t.Errorf("stream-5 should have been scrubbed") + } + + files, err := os.ReadDir(cacheDir) + if err != nil { + t.Error(err.Error()) + return + } + + if len(files) != 4 { + t.Errorf("expected 4 items in directory") + } +} + +func createCachedStream(c *fscache.FSCache, name string, contents string) fscache.ReadAtCloser { + r, w, _ := c.Get(name) + _, _ = w.Write([]byte(contents)) + _ = w.Close() + _, _ = io.Copy(io.Discard, r) + return r +}