third-party-mirrors/navidrome
1
0
Fork 0
mirror of https://github.com/navidrome/navidrome.git synced 2025-04-15 11:40:36 +03:00
Code Issues Packages Projects Releases Wiki Activity

feat: make rescan faster, only loading metadata from changed files

Browse Source
This commit is contained in:
Deluan 2020-02-13 20:18:17 -05:00
parent d9993c5877
commit ba08f00c20
3 changed files with 39 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scanner/metadata_ffmpeg.go
View File

@ -43,7 +43,7 @@ func (m *Metadata) FilePath() string { return m.filePath }
func (m *Metadata) Suffix() string { return m.suffix }
func (m *Metadata) Size() int { return int(m.fileInfo.Size()) }
func LoadAllAudioFiles(dirPath string) ([]os.FileInfo, error) {
func LoadAllAudioFiles(dirPath string) (map[string]os.FileInfo, error) {
dir, err := os.Open(dirPath)
if err != nil {
return nil, err
@ -52,7 +52,7 @@ func LoadAllAudioFiles(dirPath string) ([]os.FileInfo, error) {
if err != nil {
return nil, err
}
var audioFiles []os.FileInfo
audioFiles := make(map[string]os.FileInfo)
for _, f := range files {
if f.IsDir() {
continue
@ -66,7 +66,7 @@ func LoadAllAudioFiles(dirPath string) ([]os.FileInfo, error) {
if err != nil {
log.Error("Could not stat file", "filePath", filePath, err)
} else {
audioFiles = append(audioFiles, fi)
audioFiles[filePath] = fi
}
}

5
scanner/metadata_test.go
View File

@ -7,7 +7,7 @@ import (
var _ = Describe("Metadata", func() {
// TODO Need to mock `ffmpeg`
XContext("ExtractAllMetadata", func() {
Context("ExtractAllMetadata", func() {
It("correctly parses metadata from all files in folder", func() {
mds, err := ExtractAllMetadata([]string{"tests/fixtures/test.mp3", "tests/fixtures/test.ogg"})
Expect(err).NotTo(HaveOccurred())
@ -52,6 +52,9 @@ var _ = Describe("Metadata", func() {
files, err := LoadAllAudioFiles("tests/fixtures")
Expect(err).ToNot(HaveOccurred())
Expect(files).To(HaveLen(3))
Expect(files).To(HaveKey("tests/fixtures/test.ogg"))
Expect(files).To(HaveKey("tests/fixtures/test.mp3"))
Expect(files).To(HaveKey("tests/fixtures/01 Invisible (RED) Edit Version.mp3"))
})
It("returns error if path does not exist", func() {
_, err := LoadAllAudioFiles("./INVALID/PATH")

61
scanner/tag_scanner.go
View File

@ -143,44 +143,57 @@ func (s *TagScanner) processChangedDir(ctx context.Context, dir string, updatedA
return err
}
for _, t := range ct {
currentTracks[t.ID] = t
currentTracks[t.Path] = t
}
// Load tracks from the folder
newTracks, err := s.loadTracks(dir)
// Load tracks FileInfo from the folder
files, err := LoadAllAudioFiles(dir)
if err != nil {
return err
}
// If no tracks to process, return
if len(newTracks)+len(currentTracks) == 0 {
// If no files to process, return
if len(files)+len(currentTracks) == 0 {
return nil
}
// If track from folder is newer than the one in DB, select for update/insert in DB and delete from the current tracks
log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(files))
var filesToUpdate []string
for filePath, info := range files {
c, ok := currentTracks[filePath]
if !ok || (ok && info.ModTime().After(c.UpdatedAt)) {
filesToUpdate = append(filesToUpdate, filePath)
}
delete(currentTracks, filePath)
}
// Load tracks Metadata from the folder
newTracks, err := s.loadTracks(filesToUpdate)
if err != nil {
return err
}
// If track from folder is newer than the one in DB, update/insert in DB and delete from the current tracks
log.Trace("Processing changed folder", "dir", dir, "tracksInDB", len(currentTracks), "tracksInFolder", len(newTracks))
log.Trace("Updating mediaFiles in DB", "dir", dir, "files", filesToUpdate, "numFiles", len(filesToUpdate))
numUpdatedTracks := 0
numPurgedTracks := 0
for _, n := range newTracks {
c, ok := currentTracks[n.ID]
if !ok || (ok && n.UpdatedAt.After(c.UpdatedAt)) {
err := s.ds.MediaFile(ctx).Put(&n)
updatedArtists[n.ArtistID] = true
updatedAlbums[n.AlbumID] = true
numUpdatedTracks++
if err != nil {
return err
}
err := s.ds.MediaFile(ctx).Put(&n)
updatedArtists[n.ArtistID] = true
updatedAlbums[n.AlbumID] = true
numUpdatedTracks++
if err != nil {
return err
}
delete(currentTracks, n.ID)
}
// Remaining tracks from DB that are not in the folder are deleted
for id, ct := range currentTracks {
for _, ct := range currentTracks {
numPurgedTracks++
updatedArtists[ct.ArtistID] = true
updatedAlbums[ct.AlbumID] = true
if err := s.ds.MediaFile(ctx).Delete(id); err != nil {
if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {
return err
}
}
@ -206,17 +219,7 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, dir string, updatedA
return s.ds.MediaFile(ctx).DeleteByPath(dir)
}
func (s *TagScanner) loadTracks(dirPath string) (model.MediaFiles, error) {
audioFiles, err := LoadAllAudioFiles(dirPath)
if err != nil {
return nil, err
}
filePaths := make([]string, len(audioFiles))
for i, _ := range audioFiles {
filePaths[i] = filepath.Join(dirPath, audioFiles[i].Name())
}
func (s *TagScanner) loadTracks(filePaths []string) (model.MediaFiles, error) {
mds, err := ExtractAllMetadata(filePaths)
if err != nil {
return nil, err