diff --git a/domain/artist.go b/domain/artist.go index 66bd2f407..8f9777aa3 100644 --- a/domain/artist.go +++ b/domain/artist.go @@ -11,3 +11,5 @@ type ArtistRepository interface { Get(id string) (*Artist, error) GetByName(name string) (*Artist, error) } + +type Artists []Artist diff --git a/persistence/album_repository.go b/persistence/album_repository.go index d28c513a7..d4595a76c 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -1,6 +1,7 @@ package persistence import ( + "errors" "github.com/deluan/gosonic/domain" ) @@ -16,7 +17,7 @@ func NewAlbumRepository() domain.AlbumRepository { func (r *albumRepository) Put(m *domain.Album) error { if m.Id == "" { - m.Id = r.NewId(m.ArtistId, m.Name) + return errors.New("Album Id is not set") } return r.saveOrUpdate(m.Id, m) } diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index b61899633..7f096bfc4 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -1,6 +1,7 @@ package persistence import ( + "errors" "github.com/deluan/gosonic/domain" ) @@ -16,7 +17,7 @@ func NewArtistRepository() domain.ArtistRepository { func (r *artistRepository) Put(m *domain.Artist) error { if m.Id == "" { - m.Id = r.NewId(m.Name) + return errors.New("Artist Id is not set") } return r.saveOrUpdate(m.Id, m) } @@ -32,4 +33,4 @@ func (r *artistRepository) GetByName(name string) (*domain.Artist, error) { return r.Get(id) } -var _ domain.ArtistRepository = (*artistRepository)(nil) \ No newline at end of file +var _ domain.ArtistRepository = (*artistRepository)(nil) diff --git a/persistence/index_repository.go b/persistence/index_repository.go index 682ad149e..41ffbf1b1 100644 --- a/persistence/index_repository.go +++ b/persistence/index_repository.go @@ -18,7 +18,7 @@ func NewArtistIndexRepository() domain.ArtistIndexRepository { func (r *artistIndexRepository) Put(m *domain.ArtistIndex) error { if m.Id == "" { - return errors.New("Id is not set") + return errors.New("Index Id is not set") } sort.Sort(m.Artists) return r.saveOrUpdate(m.Id, m) @@ -32,8 +32,8 @@ func (r *artistIndexRepository) Get(id string) (*domain.ArtistIndex, error) { func (r *artistIndexRepository) GetAll() (domain.ArtistIndexes, error) { var indices = make(domain.ArtistIndexes, 0) - err := r.loadAll(&indices, domain.QueryOptions{Alpha:true}) + err := r.loadAll(&indices, domain.QueryOptions{Alpha: true}) return indices, err } -var _ domain.ArtistIndexRepository = (*artistIndexRepository)(nil) \ No newline at end of file +var _ domain.ArtistIndexRepository = (*artistIndexRepository)(nil) diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index 217c5bbeb..12a7973ef 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -1,6 +1,7 @@ package persistence import ( + "errors" "github.com/deluan/gosonic/domain" "sort" ) @@ -16,6 +17,9 @@ func NewMediaFileRepository() domain.MediaFileRepository { } func (r *mediaFileRepository) Put(m *domain.MediaFile) error { + if m.Id == "" { + return errors.New("MediaFile Id is not set") + } return r.saveOrUpdate(m.Id, m) } diff --git a/scanner/importer.go b/scanner/importer.go index 45215fd25..5519bcf0c 100644 --- a/scanner/importer.go +++ b/scanner/importer.go @@ -7,14 +7,15 @@ import ( "github.com/deluan/gosonic/domain" "github.com/deluan/gosonic/persistence" "github.com/deluan/gosonic/utils" - "github.com/dhowden/tag" - "os" "strings" "time" ) type Scanner interface { - LoadFolder(path string) []Track + ScanLibrary(path string) (int, error) + MediaFiles() map[string]*domain.MediaFile + Albums() map[string]*domain.Album + Artists() map[string]*domain.Artist } type tempIndex map[string]domain.ArtistInfo @@ -47,19 +48,43 @@ type Importer struct { func (i *Importer) Run() { beego.Info("Starting iTunes import from:", i.mediaFolder) - files := i.scanner.LoadFolder(i.mediaFolder) - i.importLibrary(files) - beego.Info("Finished importing", len(files), "files") + if total, err := i.scanner.ScanLibrary(i.mediaFolder); err != nil { + beego.Error("Error importing iTunes Library:", err) + return + } else { + //fmt.Printf(">>>>>>>>>>>>>>>>>>\n%#v\n>>>>>>>>>>>>>>>>>\n", i.scanner.Albums()) + beego.Info("Found", total, "tracks,", + len(i.scanner.MediaFiles()), "songs,", + len(i.scanner.Albums()), "albums,", + len(i.scanner.Artists()), "artists") + } + if err := i.importLibrary(); err != nil { + beego.Error("Error persisting data:", err) + } + beego.Info("Finished importing tracks from iTunes Library") } -func (i *Importer) importLibrary(files []Track) (err error) { +func (i *Importer) importLibrary() (err error) { indexGroups := utils.ParseIndexGroups(beego.AppConfig.String("indexGroups")) - var artistIndex = make(map[string]tempIndex) + artistIndex := make(map[string]tempIndex) - for _, t := range files { - mf, album, artist := i.parseTrack(&t) - i.persist(mf, album, artist) - i.collectIndex(indexGroups, artist, artistIndex) + for _, mf := range i.scanner.MediaFiles() { + if err := i.mfRepo.Put(mf); err != nil { + beego.Error(err) + } + } + + for _, al := range i.scanner.Albums() { + if err := i.albumRepo.Put(al); err != nil { + beego.Error(err) + } + } + + for _, ar := range i.scanner.Artists() { + if err := i.artistRepo.Put(ar); err != nil { + beego.Error(err) + } + i.collectIndex(indexGroups, ar, artistIndex) } if err = i.saveIndex(artistIndex); err != nil { @@ -82,72 +107,6 @@ func (i *Importer) importLibrary(files []Track) (err error) { return err } -func (i *Importer) hasCoverArt(path string) bool { - if _, err := os.Stat(path); err == nil { - f, err := os.Open(path) - if err != nil { - beego.Warn("Error opening file", path, "-", err) - return false - } - defer f.Close() - - m, err := tag.ReadFrom(f) - if err != nil { - beego.Warn("Error reading tag from file", path, "-", err) - } - return m.Picture() != nil - } - //beego.Warn("File not found:", path) - return false -} - -func (i *Importer) parseTrack(t *Track) (*domain.MediaFile, *domain.Album, *domain.Artist) { - hasCover := i.hasCoverArt(t.Path) - mf := &domain.MediaFile{ - Id: t.Id, - Album: t.Album, - Artist: t.Artist, - AlbumArtist: t.AlbumArtist, - Title: t.Title, - Compilation: t.Compilation, - Starred: t.Loved, - Path: t.Path, - CreatedAt: t.CreatedAt, - UpdatedAt: t.UpdatedAt, - HasCoverArt: hasCover, - TrackNumber: t.TrackNumber, - DiscNumber: t.DiscNumber, - Genre: t.Genre, - Year: t.Year, - Size: t.Size, - Suffix: t.Suffix, - Duration: t.Duration, - BitRate: t.BitRate, - } - - album := &domain.Album{ - Name: t.Album, - Year: t.Year, - Compilation: t.Compilation, - Starred: t.AlbumLoved, - Genre: t.Genre, - Artist: t.Artist, - AlbumArtist: t.AlbumArtist, - CreatedAt: t.CreatedAt, // TODO Collect all songs for an album first - UpdatedAt: t.UpdatedAt, - } - - if mf.HasCoverArt { - album.CoverArtId = mf.Id - } - - artist := &domain.Artist{ - Name: t.RealArtist(), - } - - return mf, album, artist -} - func (i *Importer) persist(mf *domain.MediaFile, album *domain.Album, artist *domain.Artist) { if err := i.artistRepo.Put(artist); err != nil { beego.Error(err) diff --git a/scanner/itunes_scanner.go b/scanner/itunes_scanner.go index 64d0d3133..139264fd3 100644 --- a/scanner/itunes_scanner.go +++ b/scanner/itunes_scanner.go @@ -1,7 +1,12 @@ package scanner import ( + "crypto/md5" + "fmt" + "github.com/astaxie/beego" + "github.com/deluan/gosonic/domain" "github.com/deluan/itl" + "github.com/dhowden/tag" "net/url" "os" "path/filepath" @@ -9,45 +14,152 @@ import ( "strings" ) -type ItunesScanner struct{} +type ItunesScanner struct { + mediaFiles map[string]*domain.MediaFile + albums map[string]*domain.Album + artists map[string]*domain.Artist +} -func (s *ItunesScanner) LoadFolder(path string) []Track { +func (s *ItunesScanner) ScanLibrary(path string) (int, error) { xml, _ := os.Open(path) - l, _ := itl.ReadFromXML(xml) + l, err := itl.ReadFromXML(xml) + if err != nil { + return 0, err + } + + s.mediaFiles = make(map[string]*domain.MediaFile) + s.albums = make(map[string]*domain.Album) + s.artists = make(map[string]*domain.Artist) - mediaFiles := make([]Track, len(l.Tracks)) i := 0 - for id, t := range l.Tracks { + for _, t := range l.Tracks { if strings.HasPrefix(t.Location, "file://") && strings.Contains(t.Kind, "audio") { - mediaFiles[i].Id = id - mediaFiles[i].Album = unescape(t.Album) - mediaFiles[i].Title = unescape(t.Name) - mediaFiles[i].Artist = unescape(t.Artist) - mediaFiles[i].AlbumArtist = unescape(t.AlbumArtist) - mediaFiles[i].Genre = unescape(t.Genre) - mediaFiles[i].Compilation = t.Compilation - mediaFiles[i].Loved = t.Loved - mediaFiles[i].AlbumLoved = t.AlbumLoved - mediaFiles[i].Year = t.Year - mediaFiles[i].TrackNumber = t.TrackNumber - mediaFiles[i].DiscNumber = t.DiscNumber - if t.Size > 0 { - mediaFiles[i].Size = strconv.Itoa(t.Size) - } - if t.TotalTime > 0 { - mediaFiles[i].Duration = t.TotalTime / 1000 - } - mediaFiles[i].BitRate = t.BitRate - path, _ = url.QueryUnescape(t.Location) - path = strings.TrimPrefix(unescape(path), "file://") - mediaFiles[i].Path = path - mediaFiles[i].Suffix = strings.TrimPrefix(filepath.Ext(path), ".") - mediaFiles[i].CreatedAt = t.DateAdded - mediaFiles[i].UpdatedAt = t.DateModified + ar := s.collectArtists(&t) + mf := s.collectMediaFiles(&t) + s.collectAlbums(&t, mf, ar) i++ } } - return mediaFiles[0:i] + return len(l.Tracks), nil +} + +func (s *ItunesScanner) MediaFiles() map[string]*domain.MediaFile { + return s.mediaFiles +} +func (s *ItunesScanner) Albums() map[string]*domain.Album { + return s.albums +} +func (s *ItunesScanner) Artists() map[string]*domain.Artist { + return s.artists +} + +func (s *ItunesScanner) collectMediaFiles(t *itl.Track) *domain.MediaFile { + mf := &domain.MediaFile{} + mf.Id = strconv.Itoa(t.TrackID) + mf.Album = unescape(t.Album) + mf.AlbumId = albumId(t) + mf.Title = unescape(t.Name) + mf.Artist = unescape(t.Artist) + mf.AlbumArtist = unescape(t.AlbumArtist) + mf.Genre = unescape(t.Genre) + mf.Compilation = t.Compilation + mf.Starred = t.Loved + mf.Year = t.Year + mf.TrackNumber = t.TrackNumber + mf.DiscNumber = t.DiscNumber + if t.Size > 0 { + mf.Size = strconv.Itoa(t.Size) + } + if t.TotalTime > 0 { + mf.Duration = t.TotalTime / 1000 + } + mf.BitRate = t.BitRate + + path, _ := url.QueryUnescape(t.Location) + path = strings.TrimPrefix(unescape(path), "file://") + mf.Path = path + mf.Suffix = strings.TrimPrefix(filepath.Ext(path), ".") + mf.HasCoverArt = hasCoverArt(path) + + mf.CreatedAt = t.DateAdded + mf.UpdatedAt = t.DateModified + + s.mediaFiles[mf.Id] = mf + + return mf +} + +func (s *ItunesScanner) collectAlbums(t *itl.Track, mf *domain.MediaFile, ar *domain.Artist) *domain.Album { + id := albumId(t) + _, found := s.albums[id] + if !found { + s.albums[id] = &domain.Album{} + } + + al := s.albums[id] + al.Id = id + al.ArtistId = ar.Id + al.Name = mf.Album + al.Year = t.Year + al.Compilation = t.Compilation + al.Starred = t.AlbumLoved + al.Genre = mf.Genre + al.Artist = mf.Artist + al.AlbumArtist = mf.AlbumArtist + + if mf.HasCoverArt { + al.CoverArtId = mf.Id + } + + if al.CreatedAt.IsZero() || t.DateAdded.Before(al.CreatedAt) { + al.CreatedAt = t.DateAdded + } + if al.UpdatedAt.IsZero() || t.DateModified.After(al.UpdatedAt) { + al.UpdatedAt = t.DateModified + } + + return al +} + +func (s *ItunesScanner) collectArtists(t *itl.Track) *domain.Artist { + id := artistId(t) + _, found := s.artists[id] + if !found { + s.artists[id] = &domain.Artist{} + } + ar := s.artists[id] + ar.Id = id + ar.Name = unescape(realArtistName(t)) + + return ar +} + +func albumId(t *itl.Track) string { + s := fmt.Sprintf("%s\\%s", realArtistName(t), t.Album) + return fmt.Sprintf("%x", md5.Sum([]byte(s))) +} + +func artistId(t *itl.Track) string { + return fmt.Sprintf("%x", md5.Sum([]byte(realArtistName(t)))) +} + +func hasCoverArt(path string) bool { + if _, err := os.Stat(path); err == nil { + f, err := os.Open(path) + if err != nil { + beego.Warn("Error opening file", path, "-", err) + return false + } + defer f.Close() + + m, err := tag.ReadFrom(f) + if err != nil { + beego.Warn("Error reading tag from file", path, "-", err) + } + return m.Picture() != nil + } + //beego.Warn("File not found:", path) + return false } func unescape(str string) string { @@ -55,4 +167,15 @@ func unescape(str string) string { return s } +func realArtistName(t *itl.Track) string { + switch { + case t.Compilation: + return "Various Artists" + case t.AlbumArtist != "": + return t.AlbumArtist + } + + return t.Artist +} + var _ Scanner = (*ItunesScanner)(nil) diff --git a/scanner/track.go b/scanner/track.go deleted file mode 100644 index 7628a0b6d..000000000 --- a/scanner/track.go +++ /dev/null @@ -1,37 +0,0 @@ -package scanner - -import ( - "time" -) - -type Track struct { - Id string - Path string - Title string - Album string - Artist string - AlbumArtist string - Genre string - TrackNumber int - DiscNumber int - Year int - Size string - Suffix string - Duration int - BitRate int - Compilation bool - Loved bool - AlbumLoved bool - CreatedAt time.Time - UpdatedAt time.Time -} - -func (m *Track) RealArtist() string { - if m.Compilation { - return "Various Artists" - } - if m.AlbumArtist != "" { - return m.AlbumArtist - } - return m.Artist -}