From 85ddd19c3d4764fe96357f721f355d7d1e8f3331 Mon Sep 17 00:00:00 2001 From: Deluan Date: Fri, 26 Feb 2016 01:32:31 -0500 Subject: [PATCH] Implemented first repository using tiedot --- .gitignore | 6 +- .gopmfile | 1 + api/get_music_folders.go | 2 +- conf/app.conf | 5 +- controllers/sync.go | 14 +++++ models/album.go | 12 ++++ models/artist.go | 6 ++ models/media_file.go | 4 +- repositories/base_repository.go | 78 ++++++++++++++++++++++++ repositories/init_database.go | 37 +++++++++++ repositories/media_file_repository.go | 30 ++++++--- repositories/media_folders_repository.go | 9 ++- routers/router.go | 1 + scanner/itunes_scanner.go | 34 +++++++++++ scanner/scanner.go | 42 +++++++++++++ scanner/track.go | 15 +++++ scanners/itunes/itl_scanner.go | 29 --------- scanners/scanner.go | 7 --- 18 files changed, 279 insertions(+), 53 deletions(-) create mode 100644 models/album.go create mode 100644 models/artist.go create mode 100644 repositories/base_repository.go create mode 100644 repositories/init_database.go create mode 100644 scanner/itunes_scanner.go create mode 100644 scanner/scanner.go create mode 100644 scanner/track.go delete mode 100644 scanners/itunes/itl_scanner.go delete mode 100644 scanners/scanner.go diff --git a/.gitignore b/.gitignore index 33dd9d1bf..edaa77e37 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ lastupdate.tmp gosonic -iTunes Music Library.xml +iTunes*.xml gosonic.index -static/Jamstash \ No newline at end of file +static/Jamstash +devDb +tmp \ No newline at end of file diff --git a/.gopmfile b/.gopmfile index 3a807fd0c..e61b20efd 100644 --- a/.gopmfile +++ b/.gopmfile @@ -5,6 +5,7 @@ path = github.com/deluan/gosonic github.com/astaxie/beego = tag:v1.6.0 github.com/blevesearch/bleve = commit:a5bb81e github.com/dhowden/itl = commit:35d15a3 +github.com/HouzuoGuo/tiedot = tag:3.2 [res] include = conf \ No newline at end of file diff --git a/api/get_music_folders.go b/api/get_music_folders.go index 41a7237ad..d0b8153c3 100644 --- a/api/get_music_folders.go +++ b/api/get_music_folders.go @@ -9,7 +9,7 @@ import ( type GetMusicFoldersController struct{ beego.Controller } func (c *GetMusicFoldersController) Get() { - repository := new(repositories.MediaFolderRepository) + repository := repositories.NewMediaFolderRepository() mediaFolderList, _ := repository.GetAll() folders := make([]responses.MusicFolder, len(mediaFolderList)) for i, f := range mediaFolderList { diff --git a/conf/app.conf b/conf/app.conf index 5223d5ef5..c3418695e 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -5,9 +5,10 @@ autoRender = false copyRequestBody = true apiVersion = 1.0.0 -musicFolder=. +musicFolder=./iTunes.xml user=deluan password=wordpass +dbPath = ./devDb [dev] disableValidation = true @@ -16,6 +17,8 @@ indexPath = ./gosonic.index [test] disableValidation = false +httpPort = 8081 enableAdmin = false user=deluan password=wordpass +dbPath = ./tmp/testDb \ No newline at end of file diff --git a/controllers/sync.go b/controllers/sync.go index 2d3293679..e401b0d8a 100644 --- a/controllers/sync.go +++ b/controllers/sync.go @@ -1 +1,15 @@ package controllers + +import ( + "github.com/astaxie/beego" + "github.com/deluan/gosonic/scanner" +) + +type SyncController struct{ beego.Controller } + +func (c *SyncController) Get() { + scanner.StartImport() + c.Ctx.WriteString("Import started. Check logs") +} + + diff --git a/models/album.go b/models/album.go new file mode 100644 index 000000000..50fbfd5ad --- /dev/null +++ b/models/album.go @@ -0,0 +1,12 @@ +package models + +type Album struct { + Id string + Name string + Artist *Artist + CoverArtPath string + Year int + Compilation bool + Rating int + +} diff --git a/models/artist.go b/models/artist.go new file mode 100644 index 000000000..9c9360ce6 --- /dev/null +++ b/models/artist.go @@ -0,0 +1,6 @@ +package models + +type Artist struct { + Id string + Name string +} \ No newline at end of file diff --git a/models/media_file.go b/models/media_file.go index 0f210ff24..9e9b8ca6a 100644 --- a/models/media_file.go +++ b/models/media_file.go @@ -1,8 +1,6 @@ package models -import ( - "time" -) +import "time" type MediaFile struct { Id string diff --git a/repositories/base_repository.go b/repositories/base_repository.go new file mode 100644 index 000000000..0fdec44bd --- /dev/null +++ b/repositories/base_repository.go @@ -0,0 +1,78 @@ +package repositories + +import ( + "encoding/json" + "github.com/HouzuoGuo/tiedot/db" + "github.com/astaxie/beego" + "fmt" +) + +type BaseRepository struct { + col *db.Col +} + +func (r *BaseRepository) marshal(rec interface{}) (map[string]interface{}, error) { + // Convert to JSON... + b, err := json.Marshal(rec); + if err != nil { + return nil, err + } + + // ... then convert to map + var m map[string]interface{} + err = json.Unmarshal(b, &m) + return m, err +} + +func (r*BaseRepository) query(q string, a ...interface{}) (map[int]struct{}, error) { + q = fmt.Sprintf(q, a) + + var query interface{} + json.Unmarshal([]byte(q), &query) + + queryResult := make(map[int]struct{}) + + err := db.EvalQuery(query, r.col, &queryResult) + if err != nil { + beego.Warn("Error '%s' - query='%s'", q, err) + } + return queryResult, err +} + +func (r*BaseRepository) queryFirstKey(q string, a ...interface{}) (int, error) { + result, err := r.query(q, a) + if err != nil { + return 0, err + } + for key, _ := range result { + return key, nil + } + + return 0, nil +} + +func (r *BaseRepository) saveOrUpdate(rec interface{}) error { + m, err := r.marshal(rec) + if err != nil { + return err + } + docId, err := r.queryFirstKey(`{"in": ["Id"], "eq": "%s"}`, m["Id"]) + if docId == 0 { + _, err = r.col.Insert(m) + return err + } + err = r.col.Update(docId, m) + if err != nil { + beego.Warn("Error updating %s[%d]: %s", r.col, docId, err) + } + return err +} + +func (r *BaseRepository) Dump() { + r.col.ForEachDoc(func(id int, docContent []byte) (willMoveOn bool) { + beego.Debug("Document", id, "=", string(docContent)) + return true + }) +} + + diff --git a/repositories/init_database.go b/repositories/init_database.go new file mode 100644 index 000000000..42d662126 --- /dev/null +++ b/repositories/init_database.go @@ -0,0 +1,37 @@ +package repositories + +import ( + "github.com/HouzuoGuo/tiedot/db" + "github.com/astaxie/beego" + "sync" +) + +var ( + _dbInstance *db.DB + once sync.Once +) + +func createCollection(name string) *db.Col { + col := dbInstance().Use(name) + if col != nil { + return col + } + if err := dbInstance().Create(name); err != nil { + beego.Error(err) + } + if err := col.Index([]string{"Id"}); err != nil { + beego.Error(name, err) + } + return col +} + +func dbInstance() *db.DB { + once.Do(func() { + instance, err := db.OpenDB(beego.AppConfig.String("dbPath")) + if err != nil { + panic(err) + } + _dbInstance = instance + }) + return _dbInstance +} \ No newline at end of file diff --git a/repositories/media_file_repository.go b/repositories/media_file_repository.go index df5e8c351..3a60b257b 100644 --- a/repositories/media_file_repository.go +++ b/repositories/media_file_repository.go @@ -1,10 +1,24 @@ package repositories -//import "github.com/deluan/gosonic/models" -// -//func AddMediaFile(m models.MediaFile) string { -// m.ID = "user_" + strconv.FormatInt(time.Now().UnixNano(), 10) -// UserList[u.Id] = &u -// return u.Id -//} -// +import ( + "github.com/deluan/gosonic/models" + "fmt" + "crypto/md5" +) + +type MediaFile struct { + BaseRepository +} + +func NewMediaFileRepository() *MediaFile { + r := &MediaFile{} + r.col = createCollection("MediaFiles") + return r +} + +func (r *MediaFile) Add(m *models.MediaFile) error { + if m.Id == "" { + m.Id = fmt.Sprintf("%x", md5.Sum([]byte(m.Path))) + } + return r.saveOrUpdate(m) +} \ No newline at end of file diff --git a/repositories/media_folders_repository.go b/repositories/media_folders_repository.go index 67e2f5a13..d6111f7e9 100644 --- a/repositories/media_folders_repository.go +++ b/repositories/media_folders_repository.go @@ -5,9 +5,14 @@ import ( "github.com/astaxie/beego" ) -type MediaFolderRepository struct {} +type MediaFolder struct {} -func (*MediaFolderRepository) GetAll() ([]*models.MediaFolder, error) { +func NewMediaFolderRepository() *MediaFolder { + return &MediaFolder{} +} + + +func (*MediaFolder) GetAll() ([]*models.MediaFolder, error) { mediaFolder := models.MediaFolder{Id: "0", Name: "iTunes Library", Path: beego.AppConfig.String("musicFolder")} result := make([]*models.MediaFolder, 1) result[0] = &mediaFolder diff --git a/routers/router.go b/routers/router.go index f3848dde9..4ba45230c 100644 --- a/routers/router.go +++ b/routers/router.go @@ -17,6 +17,7 @@ func init() { beego.AddNamespace(ns) beego.Router("/", &controllers.MainController{}) + beego.Router("/sync", &controllers.SyncController{}) var ValidateRequest = func(ctx *context.Context) { api.Validate(&beego.Controller{Ctx: ctx}) diff --git a/scanner/itunes_scanner.go b/scanner/itunes_scanner.go new file mode 100644 index 000000000..f8c1ec383 --- /dev/null +++ b/scanner/itunes_scanner.go @@ -0,0 +1,34 @@ +package scanner + +import ( + "github.com/dhowden/itl" + "net/url" + "os" + "strings" +) + +type ItunesScanner struct {} + +func (s *ItunesScanner) LoadFolder(path string) []Track { + xml, _ := os.Open(path) + l, _ := itl.ReadFromXML(xml) + + mediaFiles := make([]Track, len(l.Tracks)) + i := 0 + for id, t := range l.Tracks { + if t.Location != "" && strings.Contains(t.Kind, "audio") { + mediaFiles[i].Id = id + mediaFiles[i].Album = t.Album + mediaFiles[i].Title = t.Name + mediaFiles[i].Artist = t.Artist + path, _ = url.QueryUnescape(t.Location) + mediaFiles[i].Path = strings.TrimPrefix(path, "file://") + mediaFiles[i].CreatedAt = t.DateAdded + mediaFiles[i].UpdatedAt = t.DateModified + i++ + } + } + return mediaFiles[0:i] +} + +var _ Scanner = (*ItunesScanner)(nil) \ No newline at end of file diff --git a/scanner/scanner.go b/scanner/scanner.go new file mode 100644 index 000000000..be690d795 --- /dev/null +++ b/scanner/scanner.go @@ -0,0 +1,42 @@ +package scanner + +import ( + "github.com/astaxie/beego" + "github.com/deluan/gosonic/repositories" + "github.com/deluan/gosonic/models" +) + +type Scanner interface { + LoadFolder(path string) []Track +} + +func StartImport() { + go doImport(beego.AppConfig.String("musicFolder"), &ItunesScanner{}) +} + +func doImport(mediaFolder string, scanner Scanner) { + beego.Info("Starting iTunes import from:", mediaFolder) + files := scanner.LoadFolder(mediaFolder) + updateDatastore(files) + beego.Info("Finished importing", len(files), "files") +} + +func updateDatastore(files []Track) { + mfRepo := repositories.NewMediaFileRepository() + for _, t := range files { + m := &models.MediaFile{ + Id: t.Id, + Album: t.Album, + Artist: t.Artist, + Title: t.Title, + Path: t.Path, + CreatedAt: t.CreatedAt, + UpdatedAt: t.UpdatedAt, + } + err := mfRepo.Add(m) + if err != nil { + beego.Error(err) + } + } + mfRepo.Dump() +} \ No newline at end of file diff --git a/scanner/track.go b/scanner/track.go new file mode 100644 index 000000000..c87e37212 --- /dev/null +++ b/scanner/track.go @@ -0,0 +1,15 @@ +package scanner + +import ( + "time" +) + +type Track struct { + Id string + Path string + Album string + Artist string + Title string + CreatedAt time.Time + UpdatedAt time.Time +} \ No newline at end of file diff --git a/scanners/itunes/itl_scanner.go b/scanners/itunes/itl_scanner.go deleted file mode 100644 index b26e2b98d..000000000 --- a/scanners/itunes/itl_scanner.go +++ /dev/null @@ -1,29 +0,0 @@ -package itunes - -import ( - "github.com/deluan/gosonic/models" - "github.com/dhowden/itl" - "net/url" - "os" - "strings" -) - -func LoadFolder(path string) []models.MediaFile { - xml, _ := os.Open(path) - l, _ := itl.ReadFromXML(xml) - - mediaFiles := make([]models.MediaFile, len(l.Tracks)) - i := 0 - for id, track := range l.Tracks { - mediaFiles[i].Id = id - mediaFiles[i].Album = track.Album - mediaFiles[i].Title = track.Name - mediaFiles[i].Artist = track.Artist - path, _ = url.QueryUnescape(track.Location) - mediaFiles[i].Path = strings.TrimPrefix(path, "file://") - mediaFiles[i].CreatedAt = track.DateAdded - mediaFiles[i].UpdatedAt = track.DateModified - i++ - } - return mediaFiles -} diff --git a/scanners/scanner.go b/scanners/scanner.go deleted file mode 100644 index af7b7be7b..000000000 --- a/scanners/scanner.go +++ /dev/null @@ -1,7 +0,0 @@ -package scanners - -import "github.com/deluan/gosonic/models" - -type Scanner interface { - LoadFolder(path string) []models.MediaFile -}