From 14e52576a7f9242f8a928eae228b0f50c3a69f78 Mon Sep 17 00:00:00 2001
From: Deluan <dquintao@adparlor.com>
Date: Sun, 28 Feb 2016 13:50:05 -0500
Subject: [PATCH] Scanning artists and albums too
---
models/album.go | 36 ++++++--
models/artist.go | 21 ++++-
repositories/album_repository.go | 34 ++++++++
repositories/artist_repository.go | 34 ++++++++
repositories/base_repository.go | 18 +++-
.../{ledis_database.go => ledis_utils.go} | 12 +--
repositories/media_file_repository.go | 7 +-
scanner/itunes_scanner.go | 1 +
scanner/scanner.go | 87 +++++++++++++++----
scanner/track.go | 1 +
utils/mapping.go | 9 +-
11 files changed, 209 insertions(+), 51 deletions(-)
create mode 100644 repositories/album_repository.go
create mode 100644 repositories/artist_repository.go
rename repositories/{ledis_database.go => ledis_utils.go} (83%)
diff --git a/models/album.go b/models/album.go
index 50fbfd5ad..81179230a 100644
--- a/models/album.go
+++ b/models/album.go
@@ -1,12 +1,34 @@
package models
type Album struct {
- Id string
- Name string
- Artist *Artist
+ Id string
+ Name string
+ ArtistId string
CoverArtPath string
- Year int
- Compilation bool
- Rating int
-
+ Year int
+ Compilation bool
+ Rating int
+ MediaFiles map[string]bool
}
+
+func (a *Album) AdMediaFiles(mfs ...*MediaFile) {
+ for _, mf := range mfs {
+ a.MediaFiles[mf.Id] = true
+ }
+}
+
+func (a *Album) AddMediaFiles(mfs ...interface{}) {
+ if a.MediaFiles == nil {
+ a.MediaFiles = make(map[string]bool)
+ }
+ for _, v := range mfs {
+ switch v := v.(type) {
+ case *MediaFile:
+ a.MediaFiles[v.Id] = true
+ case map[string]bool:
+ for k, _ := range v {
+ a.MediaFiles[k] = true
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/models/artist.go b/models/artist.go
index 7637bcff5..43247ae51 100644
--- a/models/artist.go
+++ b/models/artist.go
@@ -6,8 +6,9 @@ import (
)
type Artist struct {
- Id string
- Name string
+ Id string
+ Name string
+ Albums map[string]bool
}
func NoArticle(name string) string {
@@ -19,4 +20,20 @@ func NoArticle(name string) string {
}
}
return name
+}
+
+func (a *Artist) AddAlbums(albums ...interface{}) {
+ if a.Albums == nil {
+ a.Albums = make(map[string]bool)
+ }
+ for _, v := range albums {
+ switch v := v.(type) {
+ case *Album:
+ a.Albums[v.Id] = true
+ case map[string]bool:
+ for k, _ := range v {
+ a.Albums[k] = true
+ }
+ }
+ }
}
\ No newline at end of file
diff --git a/repositories/album_repository.go b/repositories/album_repository.go
new file mode 100644
index 000000000..18a460c46
--- /dev/null
+++ b/repositories/album_repository.go
@@ -0,0 +1,34 @@
+package repositories
+
+import (
+ "github.com/deluan/gosonic/models"
+)
+
+type Album struct {
+ BaseRepository
+}
+
+func NewAlbumRepository() *Album {
+ r := &Album{}
+ r.key = "album"
+ return r
+}
+
+func (r *Album) Put(m *models.Album) (*models.Album, error) {
+ if m.Id == "" {
+ m.Id = r.NewId(m.Name)
+ }
+ return m, r.saveOrUpdate(m.Id, m)
+}
+
+func (r *Album) Get(id string) (*models.Album, error) {
+ rec := &models.Album{}
+ err := readStruct(r.key, id, rec)
+ return rec, err
+}
+
+func (r *Album) GetByName(name string) (*models.Album, error) {
+ id := r.NewId(name)
+ return r.Get(id)
+}
+
diff --git a/repositories/artist_repository.go b/repositories/artist_repository.go
new file mode 100644
index 000000000..b67919c06
--- /dev/null
+++ b/repositories/artist_repository.go
@@ -0,0 +1,34 @@
+package repositories
+
+import (
+ "github.com/deluan/gosonic/models"
+)
+
+type Artist struct {
+ BaseRepository
+}
+
+func NewArtistRepository() *Artist {
+ r := &Artist{}
+ r.key = "artist"
+ return r
+}
+
+func (r *Artist) Put(m *models.Artist) (*models.Artist, error) {
+ if m.Id == "" {
+ m.Id = r.NewId(m.Name)
+ }
+ return m, r.saveOrUpdate(m.Id, m)
+}
+
+func (r *Artist) Get(id string) (*models.Artist, error) {
+ rec := &models.Artist{}
+ err := readStruct(r.key, id, rec)
+ return rec, err
+}
+
+func (r *Artist) GetByName(name string) (*models.Artist, error) {
+ id := r.NewId(name)
+ return r.Get(id)
+}
+
diff --git a/repositories/base_repository.go b/repositories/base_repository.go
index a9882a597..d5057d566 100644
--- a/repositories/base_repository.go
+++ b/repositories/base_repository.go
@@ -1,18 +1,28 @@
package repositories
+import (
+ "fmt"
+ "crypto/md5"
+ "strings"
+)
+
type BaseRepository struct {
- key string
+ key string // TODO Rename to 'table'
}
-
-func (r *BaseRepository) saveOrUpdate(id string, rec interface{}) error {
- return saveStruct(r.key, id, rec)
+func (r *BaseRepository) NewId(fields ...string) string {
+ s := fmt.Sprintf("%s\\%s", strings.ToUpper(r.key), strings.Join(fields, ""))
+ return fmt.Sprintf("%x", md5.Sum([]byte(s)))
}
func (r *BaseRepository) CountAll() (int, error) {
return count(r.key)
}
+func (r *BaseRepository) saveOrUpdate(id string, rec interface{}) error {
+ return saveStruct(r.key, id, rec)
+}
+
func (r *BaseRepository) Dump() {
}
diff --git a/repositories/ledis_database.go b/repositories/ledis_utils.go
similarity index 83%
rename from repositories/ledis_database.go
rename to repositories/ledis_utils.go
index f5709aa40..9cc01db77 100644
--- a/repositories/ledis_database.go
+++ b/repositories/ledis_utils.go
@@ -46,8 +46,9 @@ func saveStruct(key, id string, data interface{}) error {
return db().HMset([]byte(kh), fvList...)
}
-func readStruct(key string) (interface{}, error) {
- fvs, _ := db().HGetAll([]byte(key))
+func readStruct(key, id string, rec interface{}) error {
+ kh := key + "_id_" + id
+ fvs, _ := db().HGetAll([]byte(kh))
var m = make(map[string]interface{}, len(fvs))
for _, fv := range fvs {
var v interface{}
@@ -55,15 +56,10 @@ func readStruct(key string) (interface{}, error) {
m[string(fv.Field)] = v
}
- return utils.ToStruct(m)
+ return utils.ToStruct(m, rec)
}
func count(key string) (int, error) {
ids, err := db().SMembers([]byte(key + "_ids"))
return len(ids), err
-}
-
-func hset(key, field, value string) error {
- _, err := db().HSet([]byte(key), []byte(field), []byte(value))
- return err
}
\ No newline at end of file
diff --git a/repositories/media_file_repository.go b/repositories/media_file_repository.go
index 752fb31fa..e0869e5c1 100644
--- a/repositories/media_file_repository.go
+++ b/repositories/media_file_repository.go
@@ -2,8 +2,6 @@ package repositories
import (
"github.com/deluan/gosonic/models"
- "fmt"
- "crypto/md5"
)
type MediaFile struct {
@@ -16,9 +14,6 @@ func NewMediaFileRepository() *MediaFile {
return r
}
-func (r *MediaFile) Add(m *models.MediaFile) error {
- if m.Id == "" {
- m.Id = fmt.Sprintf("%x", md5.Sum([]byte(m.Path)))
- }
+func (r *MediaFile) Put(m *models.MediaFile) error {
return r.saveOrUpdate(m.Id, m)
}
\ No newline at end of file
diff --git a/scanner/itunes_scanner.go b/scanner/itunes_scanner.go
index a60bdf399..c15d14791 100644
--- a/scanner/itunes_scanner.go
+++ b/scanner/itunes_scanner.go
@@ -24,6 +24,7 @@ func (s *ItunesScanner) LoadFolder(path string) []Track {
mediaFiles[i].Artist = t.Artist
mediaFiles[i].AlbumArtist = t.AlbumArtist
mediaFiles[i].Compilation = t.Compilation
+ mediaFiles[i].Year = t.Year
path, _ = url.QueryUnescape(t.Location)
mediaFiles[i].Path = strings.TrimPrefix(path, "file://")
mediaFiles[i].CreatedAt = t.DateAdded
diff --git a/scanner/scanner.go b/scanner/scanner.go
index 532cd5341..0cb5eff78 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -5,6 +5,7 @@ import (
"github.com/deluan/gosonic/repositories"
"github.com/deluan/gosonic/models"
"strings"
+ "fmt"
)
type Scanner interface {
@@ -18,38 +19,86 @@ func StartImport() {
func doImport(mediaFolder string, scanner Scanner) {
beego.Info("Starting iTunes import from:", mediaFolder)
files := scanner.LoadFolder(mediaFolder)
- updateDatastore(files)
+ importLibrary(files)
beego.Info("Finished importing", len(files), "files")
}
-func updateDatastore(files []Track) {
+func importLibrary(files []Track) {
mfRepo := repositories.NewMediaFileRepository()
+ albumRepo := repositories.NewAlbumRepository()
+ artistRepo := repositories.NewArtistRepository()
var artistIndex = make(map[string]map[string]string)
+
for _, t := range files {
- m := &models.MediaFile{
- Id: t.Id,
- Album: t.Album,
- Artist: t.Artist,
- AlbumArtist: t.AlbumArtist,
- Title: t.Title,
- Compilation: t.Compilation,
- Path: t.Path,
- CreatedAt: t.CreatedAt,
- UpdatedAt: t.UpdatedAt,
- }
- err := mfRepo.Add(m)
- if err != nil {
- beego.Error(err)
- }
- collectIndex(m, artistIndex)
+ mf, album, artist := processTrack(&t)
+ mergeInfo(mfRepo, mf, albumRepo, album, artistRepo, artist)
+ fmt.Printf("%#v\n", album)
+ fmt.Printf("%#v\n\n", artist)
+ collectIndex(mf, artistIndex)
}
- //mfRepo.Dump()
+
//j,_ := json.MarshalIndent(artistIndex, "", " ")
//fmt.Println(string(j))
c, _ := mfRepo.CountAll()
+
beego.Info("Total mediafiles in database:", c)
}
+func processTrack(t *Track) (*models.MediaFile, *models.Album, *models.Artist) {
+ mf := &models.MediaFile{
+ Id: t.Id,
+ Album: t.Album,
+ Artist: t.Artist,
+ AlbumArtist: t.AlbumArtist,
+ Title: t.Title,
+ Compilation: t.Compilation,
+ Path: t.Path,
+ CreatedAt: t.CreatedAt,
+ UpdatedAt: t.UpdatedAt,
+ }
+
+ album := &models.Album{
+ Name: t.Album,
+ Year: t.Year,
+ Compilation: t.Compilation,
+ }
+
+ artist := &models.Artist{
+ Name: t.Artist,
+ }
+
+ return mf, album, artist
+}
+
+func mergeInfo(mfRepo *repositories.MediaFile, mf *models.MediaFile, albumRepo *repositories.Album, album *models.Album, artistRepo *repositories.Artist, artist *models.Artist) {
+ artist.Id = artistRepo.NewId(artist.Name)
+
+ sAlbum, err := albumRepo.GetByName(album.Name)
+ if err != nil {
+ beego.Error(err)
+ }
+ album.ArtistId = artist.Id
+ album.AddMediaFiles(mf, sAlbum.MediaFiles)
+ sAlbum, err = albumRepo.Put(album)
+ if err != nil {
+ beego.Error(err)
+ }
+
+ sArtist, err := artistRepo.GetByName(artist.Name)
+ if err != nil {
+ beego.Error(err)
+ }
+ artist.AddAlbums(sAlbum, sArtist.Albums)
+ sArtist, err = artistRepo.Put(artist)
+ if err != nil {
+ beego.Error(err)
+ }
+
+ if err := mfRepo.Put(mf); err != nil {
+ beego.Error(err)
+ }
+}
+
func collectIndex(m *models.MediaFile, artistIndex map[string]map[string]string) {
name := m.RealArtist()
indexName := strings.ToLower(models.NoArticle(name))
diff --git a/scanner/track.go b/scanner/track.go
index 478c8a993..f1730d24f 100644
--- a/scanner/track.go
+++ b/scanner/track.go
@@ -11,6 +11,7 @@ type Track struct {
Album string
Artist string
AlbumArtist string
+ Year int
Compilation bool
CreatedAt time.Time
UpdatedAt time.Time
diff --git a/utils/mapping.go b/utils/mapping.go
index 3a6949f9a..f950e900f 100644
--- a/utils/mapping.go
+++ b/utils/mapping.go
@@ -121,17 +121,16 @@ func ToMap(rec interface{}) (map[string]interface{}, error) {
return m, err
}
-func ToStruct(m map[string]interface{}) (interface{}, error) {
+func ToStruct(m map[string]interface{}, rec interface{}) error {
// Convert to JSON...
b, err := json.Marshal(m);
if err != nil {
- return nil, err
+ return err
}
- // ... then convert to map
- var rec interface{}
+ // ... then convert to struct
err = json.Unmarshal(b, &rec)
- return rec, err
+ return err
}
func Flatten(input interface{}) (map[string]interface{}, error) {