mirror of
https://github.com/navidrome/navidrome.git
synced 2026-11-18 02:52:36 +03:00
* chore(lint): ban context.Background in tests, flag unused nolints Add a forbidigo rule that rejects context.Background and context.TODO in _test.go files. The lint message points to the right fix: the spec's ctx from GinkgoT().Context(), or t.Context() and b.Context() in plain tests and benchmarks. AGENTS.md already asks for this, but nothing enforced it, so new tests kept adding Background calls. All 465 existing calls are replaced, with one ctx per top-level Describe set at the top of its BeforeEach. The migration forbidigo rules keep their old scope through an exclusion matched on their message text. Also enable nolintlint with require-specific and allow-unused: false. It found 41 //nolint directives that suppressed nothing, or named no linter, or had a leading space that made them inert. They are removed or fixed. The sweep also fixed a middleware test that wrote a user with a nil ctx before ctx was set. * test: keep cleanup ROLLBACKs and the scanner singleton off the spec ctx The ROLLBACK cleanups in the library, plugin and initial setup tests must run even if the spec context is already cancelled, so they use context.WithoutCancel. The same goes for the context handed to the process-wide scanner singleton in the GetInstance test. Also document that the forbidigo exclusions are scoped by message text. * test: pass the caller's ctx to tests.ClearDB ClearDB was the last context.Background call in shared test code. It now takes a ctx, and every caller passes its spec ctx. The forbidigo rule for tests also covers the tests/ helper package now, so new helpers there cannot add Background calls either.
187 lines
6.0 KiB
Go
187 lines
6.0 KiB
Go
package cache
|
|
|
|
import (
|
|
"crypto/sha1"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/djherbis/atime"
|
|
"github.com/djherbis/fscache"
|
|
"github.com/djherbis/stream"
|
|
"github.com/navidrome/navidrome/log"
|
|
)
|
|
|
|
const completeMarkerSuffix = ".complete"
|
|
const sentinelName = ".nd-migrated"
|
|
|
|
type spreadFS struct {
|
|
root string
|
|
mode os.FileMode
|
|
init func() error
|
|
}
|
|
|
|
// NewSpreadFS returns a FileSystem rooted at directory dir. This FS hashes the key and
|
|
// distributes all files in a layout like XX/XX/XXXXXXXXXX. Ex:
|
|
//
|
|
// Key is abc123.300x300.jpg
|
|
// Hash would be: c574aeb3caafcf93ee337f0cf34e31a428ba3f13
|
|
// File in cache would be: c5 / 74 / c574aeb3caafcf93ee337f0cf34e31a428ba3f13
|
|
//
|
|
// The idea is to avoid having too many files in one dir, which could potentially cause performance issues
|
|
// and may hit limitations depending on the OS.
|
|
// See discussion here: https://github.com/djherbis/fscache/issues/8#issuecomment-614319323
|
|
//
|
|
// dir is created with specified mode if it doesn't exist.
|
|
func NewSpreadFS(dir string, mode os.FileMode) (*spreadFS, error) {
|
|
f := &spreadFS{root: dir, mode: mode, init: func() error {
|
|
return os.MkdirAll(dir, mode)
|
|
}}
|
|
return f, f.init()
|
|
}
|
|
|
|
func (sfs *spreadFS) Reload(f func(key string, name string)) error {
|
|
// On the first run after upgrade (no sentinel yet), pre-existing files have
|
|
// no completion marker. Migrate them instead of discarding them as partials,
|
|
// so a user's whole cache isn't wiped. After the sentinel exists, an unmarked
|
|
// file is a crash partial and is discarded.
|
|
sentinel := filepath.Join(sfs.root, sentinelName)
|
|
_, sErr := os.Stat(sentinel)
|
|
migrating := os.IsNotExist(sErr)
|
|
|
|
count := 0
|
|
err := sfs.walkDataFiles(func(absoluteFilePath string) {
|
|
if _, statErr := os.Stat(sfs.markerPath(absoluteFilePath)); statErr != nil {
|
|
switch {
|
|
case migrating:
|
|
if mErr := sfs.MarkComplete(absoluteFilePath); mErr != nil {
|
|
log.Warn("Error migrating cache file", "file", absoluteFilePath, mErr)
|
|
}
|
|
case os.IsNotExist(statErr):
|
|
// No completion marker: this is a partial left by a crash. Discard it.
|
|
log.Debug("Removing incomplete cache file", "file", absoluteFilePath)
|
|
_ = os.Remove(absoluteFilePath)
|
|
return
|
|
default:
|
|
// Marker may exist but is unreadable (transient I/O, permissions):
|
|
// skip adoption without destroying a possibly-valid entry.
|
|
log.Warn("Error reading cache completion marker", "file", absoluteFilePath, statErr)
|
|
return
|
|
}
|
|
}
|
|
f(absoluteFilePath, absoluteFilePath)
|
|
count++
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
log.Debug("Loaded cache", "dir", sfs.root, "numItems", count)
|
|
// Only record the migration as done after a clean walk, so a partial walk
|
|
// doesn't leave valid-but-unmarked files to be discarded on the next run.
|
|
if migrating {
|
|
if wErr := os.WriteFile(sentinel, nil, 0600); wErr != nil {
|
|
log.Warn("Error writing cache migration sentinel", "file", sentinel, wErr)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// walkDataFiles visits every cache data file (named XX/XX/<40-hex>), skipping
|
|
// completion markers and opportunistically cleaning up orphaned ones.
|
|
func (sfs *spreadFS) walkDataFiles(visit func(absoluteFilePath string)) error {
|
|
return filepath.WalkDir(sfs.root, func(absoluteFilePath string, _ fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
log.Error("Error loading cache", "dir", sfs.root, err)
|
|
return nil
|
|
}
|
|
path, err := filepath.Rel(sfs.root, absoluteFilePath)
|
|
if err != nil {
|
|
return nil //nolint:nilerr
|
|
}
|
|
|
|
// Skip marker files; also clean orphan markers (data file gone).
|
|
if strings.HasSuffix(path, completeMarkerSuffix) {
|
|
dataPath := strings.TrimSuffix(absoluteFilePath, completeMarkerSuffix)
|
|
if _, statErr := os.Stat(dataPath); os.IsNotExist(statErr) {
|
|
_ = os.Remove(absoluteFilePath) //nolint:gosec // best-effort cleanup; re-swept on next Reload
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Skip if name is not in the format XX/XX/XXXXXXXXXXXX
|
|
parts := strings.Split(path, string(os.PathSeparator))
|
|
if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 || len(parts[2]) != 40 {
|
|
return nil
|
|
}
|
|
|
|
visit(absoluteFilePath)
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (sfs *spreadFS) Create(name string) (stream.File, error) {
|
|
path := filepath.Dir(name)
|
|
err := os.MkdirAll(path, sfs.mode)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return createDataFile(name)
|
|
}
|
|
|
|
func (sfs *spreadFS) Open(name string) (stream.File, error) {
|
|
return os.Open(name)
|
|
}
|
|
|
|
func (sfs *spreadFS) markerPath(dataPath string) string {
|
|
return dataPath + completeMarkerSuffix
|
|
}
|
|
|
|
// MarkComplete records that the cache entry for key was written in full.
|
|
// Only files with a marker are adopted on the next Reload; this is what
|
|
// distinguishes a complete cache entry from a partial one left by a crash.
|
|
// key may be an original cache key or an already-mapped data path; KeyMapper
|
|
// is idempotent for the latter (see KeyMapper).
|
|
func (sfs *spreadFS) MarkComplete(key string) error {
|
|
f, err := os.OpenFile(sfs.markerPath(sfs.KeyMapper(key)), os.O_CREATE|os.O_WRONLY, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return f.Close()
|
|
}
|
|
|
|
func (sfs *spreadFS) Remove(name string) error {
|
|
if err := os.Remove(sfs.markerPath(name)); err != nil && !os.IsNotExist(err) {
|
|
log.Warn("Error removing cache completion marker", "file", name, err)
|
|
}
|
|
return os.Remove(name)
|
|
}
|
|
|
|
func (sfs *spreadFS) Stat(name string) (fscache.FileInfo, error) {
|
|
stat, err := os.Stat(name)
|
|
if err != nil {
|
|
return fscache.FileInfo{}, err
|
|
}
|
|
return fscache.FileInfo{FileInfo: stat, Atime: atime.Get(stat)}, nil
|
|
}
|
|
|
|
func (sfs *spreadFS) RemoveAll() error {
|
|
if err := os.RemoveAll(sfs.root); err != nil {
|
|
return err
|
|
}
|
|
return sfs.init()
|
|
}
|
|
|
|
func (sfs *spreadFS) KeyMapper(key string) string {
|
|
// When running the Haunter, fscache can call this KeyMapper with the cached filepath instead of the key.
|
|
// That's because we don't inform the original cache keys when reloading in the Reload function above.
|
|
// If that's the case, just return the file path, as it is the actual mapped key.
|
|
if strings.HasPrefix(key, sfs.root) {
|
|
return key
|
|
}
|
|
hash := fmt.Sprintf("%x", sha1.Sum([]byte(key)))
|
|
return filepath.Join(sfs.root, hash[0:2], hash[2:4], hash)
|
|
}
|