Big Importer/Scanner refactor

domain/artist.go

@@ -11,3 +11,5 @@ type ArtistRepository interface {
 	Get(id string) (*Artist, error)
 	GetByName(name string) (*Artist, error)
 }
+
+type Artists []Artist

persistence/album_repository.go

@@ -1,6 +1,7 @@
 package persistence
 
 import (
+	"errors"
 	"github.com/deluan/gosonic/domain"
 )
 
@@ -16,7 +17,7 @@ func NewAlbumRepository() domain.AlbumRepository {
 
 func (r *albumRepository) Put(m *domain.Album) error {
 	if m.Id == "" {
-		m.Id = r.NewId(m.ArtistId, m.Name)
+		return errors.New("Album Id is not set")
 	}
 	return r.saveOrUpdate(m.Id, m)
 }

persistence/artist_repository.go

@@ -1,6 +1,7 @@
 package persistence
 
 import (
+	"errors"
 	"github.com/deluan/gosonic/domain"
 )
 
@@ -16,7 +17,7 @@ func NewArtistRepository() domain.ArtistRepository {
 
 func (r *artistRepository) Put(m *domain.Artist) error {
 	if m.Id == "" {
-		m.Id = r.NewId(m.Name)
+		return errors.New("Artist Id is not set")
 	}
 	return r.saveOrUpdate(m.Id, m)
 }

persistence/index_repository.go

@@ -18,7 +18,7 @@ func NewArtistIndexRepository() domain.ArtistIndexRepository {
 
 func (r *artistIndexRepository) Put(m *domain.ArtistIndex) error {
 	if m.Id == "" {
-		return errors.New("Id is not set")
+		return errors.New("Index Id is not set")
 	}
 	sort.Sort(m.Artists)
 	return r.saveOrUpdate(m.Id, m)
@@ -32,7 +32,7 @@ func (r *artistIndexRepository) Get(id string) (*domain.ArtistIndex, error) {
 
 func (r *artistIndexRepository) GetAll() (domain.ArtistIndexes, error) {
 	var indices = make(domain.ArtistIndexes, 0)
-	err := r.loadAll(&indices, domain.QueryOptions{Alpha:true})
+	err := r.loadAll(&indices, domain.QueryOptions{Alpha: true})
 	return indices, err
 }
 

persistence/mediafile_repository.go

@@ -1,6 +1,7 @@
 package persistence
 
 import (
+	"errors"
 	"github.com/deluan/gosonic/domain"
 	"sort"
 )
@@ -16,6 +17,9 @@ func NewMediaFileRepository() domain.MediaFileRepository {
 }
 
 func (r *mediaFileRepository) Put(m *domain.MediaFile) error {
+	if m.Id == "" {
+		return errors.New("MediaFile Id is not set")
+	}
 	return r.saveOrUpdate(m.Id, m)
 }
 

scanner/importer.go

@@ -7,14 +7,15 @@ import (
 	"github.com/deluan/gosonic/domain"
 	"github.com/deluan/gosonic/persistence"
 	"github.com/deluan/gosonic/utils"
-	"github.com/dhowden/tag"
-	"os"
 	"strings"
 	"time"
 )
 
 type Scanner interface {
-	LoadFolder(path string) []Track
+	ScanLibrary(path string) (int, error)
+	MediaFiles() map[string]*domain.MediaFile
+	Albums() map[string]*domain.Album
+	Artists() map[string]*domain.Artist
 }
 
 type tempIndex map[string]domain.ArtistInfo
@@ -47,19 +48,43 @@ type Importer struct {
 
 func (i *Importer) Run() {
 	beego.Info("Starting iTunes import from:", i.mediaFolder)
-	files := i.scanner.LoadFolder(i.mediaFolder)
+	if total, err := i.scanner.ScanLibrary(i.mediaFolder); err != nil {
-	i.importLibrary(files)
+		beego.Error("Error importing iTunes Library:", err)
-	beego.Info("Finished importing", len(files), "files")
+		return
+	} else {
+		//fmt.Printf(">>>>>>>>>>>>>>>>>>\n%#v\n>>>>>>>>>>>>>>>>>\n", i.scanner.Albums())
+		beego.Info("Found", total, "tracks,",
+			len(i.scanner.MediaFiles()), "songs,",
+			len(i.scanner.Albums()), "albums,",
+			len(i.scanner.Artists()), "artists")
+	}
+	if err := i.importLibrary(); err != nil {
+		beego.Error("Error persisting data:", err)
+	}
+	beego.Info("Finished importing tracks from iTunes Library")
 }
 
-func (i *Importer) importLibrary(files []Track) (err error) {
+func (i *Importer) importLibrary() (err error) {
 	indexGroups := utils.ParseIndexGroups(beego.AppConfig.String("indexGroups"))
-	var artistIndex = make(map[string]tempIndex)
+	artistIndex := make(map[string]tempIndex)
 
-	for _, t := range files {
+	for _, mf := range i.scanner.MediaFiles() {
-		mf, album, artist := i.parseTrack(&t)
+		if err := i.mfRepo.Put(mf); err != nil {
-		i.persist(mf, album, artist)
+			beego.Error(err)
-		i.collectIndex(indexGroups, artist, artistIndex)
+		}
+	}
+
+	for _, al := range i.scanner.Albums() {
+		if err := i.albumRepo.Put(al); err != nil {
+			beego.Error(err)
+		}
+	}
+
+	for _, ar := range i.scanner.Artists() {
+		if err := i.artistRepo.Put(ar); err != nil {
+			beego.Error(err)
+		}
+		i.collectIndex(indexGroups, ar, artistIndex)
 	}
 
 	if err = i.saveIndex(artistIndex); err != nil {
@@ -82,72 +107,6 @@ func (i *Importer) importLibrary(files []Track) (err error) {
 	return err
 }
 
-func (i *Importer) hasCoverArt(path string) bool {
-	if _, err := os.Stat(path); err == nil {
-		f, err := os.Open(path)
-		if err != nil {
-			beego.Warn("Error opening file", path, "-", err)
-			return false
-		}
-		defer f.Close()
-
-		m, err := tag.ReadFrom(f)
-		if err != nil {
-			beego.Warn("Error reading tag from file", path, "-", err)
-		}
-		return m.Picture() != nil
-	}
-	//beego.Warn("File not found:", path)
-	return false
-}
-
-func (i *Importer) parseTrack(t *Track) (*domain.MediaFile, *domain.Album, *domain.Artist) {
-	hasCover := i.hasCoverArt(t.Path)
-	mf := &domain.MediaFile{
-		Id:          t.Id,
-		Album:       t.Album,
-		Artist:      t.Artist,
-		AlbumArtist: t.AlbumArtist,
-		Title:       t.Title,
-		Compilation: t.Compilation,
-		Starred:     t.Loved,
-		Path:        t.Path,
-		CreatedAt:   t.CreatedAt,
-		UpdatedAt:   t.UpdatedAt,
-		HasCoverArt: hasCover,
-		TrackNumber: t.TrackNumber,
-		DiscNumber:  t.DiscNumber,
-		Genre:       t.Genre,
-		Year:        t.Year,
-		Size:        t.Size,
-		Suffix:      t.Suffix,
-		Duration:    t.Duration,
-		BitRate:     t.BitRate,
-	}
-
-	album := &domain.Album{
-		Name:        t.Album,
-		Year:        t.Year,
-		Compilation: t.Compilation,
-		Starred:     t.AlbumLoved,
-		Genre:       t.Genre,
-		Artist:      t.Artist,
-		AlbumArtist: t.AlbumArtist,
-		CreatedAt:   t.CreatedAt, // TODO Collect all songs for an album first
-		UpdatedAt:   t.UpdatedAt,
-	}
-
-	if mf.HasCoverArt {
-		album.CoverArtId = mf.Id
-	}
-
-	artist := &domain.Artist{
-		Name: t.RealArtist(),
-	}
-
-	return mf, album, artist
-}
-
 func (i *Importer) persist(mf *domain.MediaFile, album *domain.Album, artist *domain.Artist) {
 	if err := i.artistRepo.Put(artist); err != nil {
 		beego.Error(err)

scanner/itunes_scanner.go

@@ -1,7 +1,12 @@
 package scanner
 
 import (
+	"crypto/md5"
+	"fmt"
+	"github.com/astaxie/beego"
+	"github.com/deluan/gosonic/domain"
 	"github.com/deluan/itl"
+	"github.com/dhowden/tag"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -9,45 +14,152 @@ import (
 	"strings"
 )
 
-type ItunesScanner struct{}
+type ItunesScanner struct {
+	mediaFiles map[string]*domain.MediaFile
+	albums     map[string]*domain.Album
+	artists    map[string]*domain.Artist
+}
 
-func (s *ItunesScanner) LoadFolder(path string) []Track {
+func (s *ItunesScanner) ScanLibrary(path string) (int, error) {
 	xml, _ := os.Open(path)
-	l, _ := itl.ReadFromXML(xml)
+	l, err := itl.ReadFromXML(xml)
+	if err != nil {
+		return 0, err
+	}
+
+	s.mediaFiles = make(map[string]*domain.MediaFile)
+	s.albums = make(map[string]*domain.Album)
+	s.artists = make(map[string]*domain.Artist)
 
-	mediaFiles := make([]Track, len(l.Tracks))
 	i := 0
-	for id, t := range l.Tracks {
+	for _, t := range l.Tracks {
 		if strings.HasPrefix(t.Location, "file://") && strings.Contains(t.Kind, "audio") {
-			mediaFiles[i].Id = id
+			ar := s.collectArtists(&t)
-			mediaFiles[i].Album = unescape(t.Album)
+			mf := s.collectMediaFiles(&t)
-			mediaFiles[i].Title = unescape(t.Name)
+			s.collectAlbums(&t, mf, ar)
-			mediaFiles[i].Artist = unescape(t.Artist)
-			mediaFiles[i].AlbumArtist = unescape(t.AlbumArtist)
-			mediaFiles[i].Genre = unescape(t.Genre)
-			mediaFiles[i].Compilation = t.Compilation
-			mediaFiles[i].Loved = t.Loved
-			mediaFiles[i].AlbumLoved = t.AlbumLoved
-			mediaFiles[i].Year = t.Year
-			mediaFiles[i].TrackNumber = t.TrackNumber
-			mediaFiles[i].DiscNumber = t.DiscNumber
-			if t.Size > 0 {
-				mediaFiles[i].Size = strconv.Itoa(t.Size)
-			}
-			if t.TotalTime > 0 {
-				mediaFiles[i].Duration = t.TotalTime / 1000
-			}
-			mediaFiles[i].BitRate = t.BitRate
-			path, _ = url.QueryUnescape(t.Location)
-			path = strings.TrimPrefix(unescape(path), "file://")
-			mediaFiles[i].Path = path
-			mediaFiles[i].Suffix = strings.TrimPrefix(filepath.Ext(path), ".")
-			mediaFiles[i].CreatedAt = t.DateAdded
-			mediaFiles[i].UpdatedAt = t.DateModified
 			i++
 		}
 	}
-	return mediaFiles[0:i]
+	return len(l.Tracks), nil
+}
+
+func (s *ItunesScanner) MediaFiles() map[string]*domain.MediaFile {
+	return s.mediaFiles
+}
+func (s *ItunesScanner) Albums() map[string]*domain.Album {
+	return s.albums
+}
+func (s *ItunesScanner) Artists() map[string]*domain.Artist {
+	return s.artists
+}
+
+func (s *ItunesScanner) collectMediaFiles(t *itl.Track) *domain.MediaFile {
+	mf := &domain.MediaFile{}
+	mf.Id = strconv.Itoa(t.TrackID)
+	mf.Album = unescape(t.Album)
+	mf.AlbumId = albumId(t)
+	mf.Title = unescape(t.Name)
+	mf.Artist = unescape(t.Artist)
+	mf.AlbumArtist = unescape(t.AlbumArtist)
+	mf.Genre = unescape(t.Genre)
+	mf.Compilation = t.Compilation
+	mf.Starred = t.Loved
+	mf.Year = t.Year
+	mf.TrackNumber = t.TrackNumber
+	mf.DiscNumber = t.DiscNumber
+	if t.Size > 0 {
+		mf.Size = strconv.Itoa(t.Size)
+	}
+	if t.TotalTime > 0 {
+		mf.Duration = t.TotalTime / 1000
+	}
+	mf.BitRate = t.BitRate
+
+	path, _ := url.QueryUnescape(t.Location)
+	path = strings.TrimPrefix(unescape(path), "file://")
+	mf.Path = path
+	mf.Suffix = strings.TrimPrefix(filepath.Ext(path), ".")
+	mf.HasCoverArt = hasCoverArt(path)
+
+	mf.CreatedAt = t.DateAdded
+	mf.UpdatedAt = t.DateModified
+
+	s.mediaFiles[mf.Id] = mf
+
+	return mf
+}
+
+func (s *ItunesScanner) collectAlbums(t *itl.Track, mf *domain.MediaFile, ar *domain.Artist) *domain.Album {
+	id := albumId(t)
+	_, found := s.albums[id]
+	if !found {
+		s.albums[id] = &domain.Album{}
+	}
+
+	al := s.albums[id]
+	al.Id = id
+	al.ArtistId = ar.Id
+	al.Name = mf.Album
+	al.Year = t.Year
+	al.Compilation = t.Compilation
+	al.Starred = t.AlbumLoved
+	al.Genre = mf.Genre
+	al.Artist = mf.Artist
+	al.AlbumArtist = mf.AlbumArtist
+
+	if mf.HasCoverArt {
+		al.CoverArtId = mf.Id
+	}
+
+	if al.CreatedAt.IsZero() || t.DateAdded.Before(al.CreatedAt) {
+		al.CreatedAt = t.DateAdded
+	}
+	if al.UpdatedAt.IsZero() || t.DateModified.After(al.UpdatedAt) {
+		al.UpdatedAt = t.DateModified
+	}
+
+	return al
+}
+
+func (s *ItunesScanner) collectArtists(t *itl.Track) *domain.Artist {
+	id := artistId(t)
+	_, found := s.artists[id]
+	if !found {
+		s.artists[id] = &domain.Artist{}
+	}
+	ar := s.artists[id]
+	ar.Id = id
+	ar.Name = unescape(realArtistName(t))
+
+	return ar
+}
+
+func albumId(t *itl.Track) string {
+	s := fmt.Sprintf("%s\\%s", realArtistName(t), t.Album)
+	return fmt.Sprintf("%x", md5.Sum([]byte(s)))
+}
+
+func artistId(t *itl.Track) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(realArtistName(t))))
+}
+
+func hasCoverArt(path string) bool {
+	if _, err := os.Stat(path); err == nil {
+		f, err := os.Open(path)
+		if err != nil {
+			beego.Warn("Error opening file", path, "-", err)
+			return false
+		}
+		defer f.Close()
+
+		m, err := tag.ReadFrom(f)
+		if err != nil {
+			beego.Warn("Error reading tag from file", path, "-", err)
+		}
+		return m.Picture() != nil
+	}
+	//beego.Warn("File not found:", path)
+	return false
 }
 
 func unescape(str string) string {
@@ -55,4 +167,15 @@ func unescape(str string) string {
 	return s
 }
 
+func realArtistName(t *itl.Track) string {
+	switch {
+	case t.Compilation:
+		return "Various Artists"
+	case t.AlbumArtist != "":
+		return t.AlbumArtist
+	}
+
+	return t.Artist
+}
+
 var _ Scanner = (*ItunesScanner)(nil)

scanner/track.go

@@ -1,37 +0,0 @@
-package scanner
-
-import (
-	"time"
-)
-
-type Track struct {
-	Id          string
-	Path        string
-	Title       string
-	Album       string
-	Artist      string
-	AlbumArtist string
-	Genre       string
-	TrackNumber int
-	DiscNumber  int
-	Year        int
-	Size        string
-	Suffix      string
-	Duration    int
-	BitRate     int
-	Compilation bool
-	Loved       bool
-	AlbumLoved  bool
-	CreatedAt   time.Time
-	UpdatedAt   time.Time
-}
-
-func (m *Track) RealArtist() string {
-	if m.Compilation {
-		return "Various Artists"
-	}
-	if m.AlbumArtist != "" {
-		return m.AlbumArtist
-	}
-	return m.Artist
-}