From 67eeb218c4321760f8635a79d61a119c6327da05 Mon Sep 17 00:00:00 2001 From: Deluan <deluan@deluan.com> Date: Sun, 19 Jan 2020 15:37:41 -0500 Subject: [PATCH] Big Refactor: - Create model.DataStore, with provision for transactions - Change all layers dependencies on repositories to use DataStore - Implemented persistence.SQLStore - Removed iTunes Bridge/Importer support --- README.md | 5 +- api/wire_gen.go | 4 +- api/wire_injectors.go | 2 - conf/configuration.go | 1 - engine/browser.go | 34 +- engine/browser_test.go | 4 +- engine/cover.go | 11 +- engine/cover_test.go | 7 +- engine/list_generator.go | 28 +- engine/playlists.go | 55 +-- engine/ratings.go | 78 +---- engine/scrobbler.go | 79 +---- engine/scrobbler_test.go | 201 ----------- engine/search.go | 14 +- itunesbridge/itunes.go | 135 -------- itunesbridge/script.go | 63 ---- model/{base.go => model.go} | 16 +- persistence/album_repository.go | 34 +- persistence/album_repository_test.go | 3 +- persistence/artist_repository.go | 30 +- persistence/artist_repository_test.go | 3 +- persistence/checksum_repository.go | 34 +- persistence/checksum_repository_test.go | 6 +- persistence/genre_repository.go | 13 +- persistence/genre_repository_test.go | 3 +- persistence/mediafile_repository.go | 46 ++- persistence/mediafile_repository_test.go | 3 +- persistence/mediafolders_repository.go | 9 +- persistence/mock_persistence.go | 54 +++ persistence/persistence.go | 66 +++- persistence/persistence_suite_test.go | 8 +- persistence/playlist_repository.go | 19 +- persistence/property_repository.go | 9 +- persistence/property_repository_test.go | 3 +- persistence/searchable_repository.go | 42 ++- persistence/sql_repository.go | 35 +- persistence/wire_provider.go | 17 +- scanner/scanner.go | 31 +- scanner/scanner_suite_test.go | 12 +- scanner/tag_scanner.go | 24 +- scanner_legacy/importer.go | 249 -------------- scanner_legacy/itunes_scanner.go | 407 ----------------------- scanner_legacy/itunes_scanner_test.go | 25 -- scanner_legacy/wire_providers.go | 9 - server/app.go | 34 +- wire_gen.go | 41 +-- wire_injectors.go | 4 - 47 files changed, 389 insertions(+), 1621 deletions(-) delete mode 100644 engine/scrobbler_test.go delete mode 100644 itunesbridge/itunes.go delete mode 100644 itunesbridge/script.go rename model/{base.go => model.go} (62%) create mode 100644 persistence/mock_persistence.go delete mode 100644 scanner_legacy/importer.go delete mode 100644 scanner_legacy/itunes_scanner.go delete mode 100644 scanner_legacy/itunes_scanner_test.go delete mode 100644 scanner_legacy/wire_providers.go diff --git a/README.md b/README.md index 442917649..9b3520504 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ CloudSonic is a music collection server and streamer, allowing you to listen to It relies on the huge selection of available mobile and web apps compatible with [Subsonic](http://www.subsonic.org), [Airsonic](https://airsonic.github.io/) and [Madsonic](https://www.madsonic.org/) -It is already functional (see [Installation](#installation) below), but still in its early stages. Currently it can only import iTunes libraries, but soon it will also be able to scan any folder with music files. +It is already functional (see [Installation](#installation) below), but still in its early stages. Version 1.0 main goals are: - Be fully compatible with available [Subsonic clients](http://www.subsonic.org/pages/apps.jsp) @@ -15,7 +15,6 @@ Version 1.0 main goals are: [DSub](http://www.subsonic.org/pages/apps.jsp#dsub), [Music Stash](https://play.google.com/store/apps/details?id=com.ghenry22.mymusicstash) and [Jamstash](http://www.subsonic.org/pages/apps.jsp#jamstash)) -- Import and use all metadata from iTunes, so that you can optionally keep using iTunes to manage your music - Implement smart/dynamic playlists (similar to iTunes) - Optimized ro run on cheap hardware (Raspberry Pi) and VPS @@ -32,7 +31,7 @@ As this is a work in progress, there are no installers yet. To have the server r the steps in the [Development Environment](#development-environment) section below, then run it with: ``` -$ export SONIC_MUSICFOLDER="/path/to/your/iTunes Library.xml" +$ export SONIC_MUSICFOLDER="/path/to/your/music/folder" $ make run ``` diff --git a/api/wire_gen.go b/api/wire_gen.go index 8071df601..391f314e1 100644 --- a/api/wire_gen.go +++ b/api/wire_gen.go @@ -6,7 +6,6 @@ package api import ( - "github.com/cloudsonic/sonic-server/itunesbridge" "github.com/google/wire" ) @@ -67,7 +66,8 @@ func initStreamController(router *Router) *StreamController { // wire_injectors.go: -var allProviders = wire.NewSet(itunesbridge.NewItunesControl, NewSystemController, +var allProviders = wire.NewSet( + NewSystemController, NewBrowsingController, NewAlbumListController, NewMediaAnnotationController, diff --git a/api/wire_injectors.go b/api/wire_injectors.go index d1997b6f4..ed96d2c42 100644 --- a/api/wire_injectors.go +++ b/api/wire_injectors.go @@ -3,12 +3,10 @@ package api import ( - "github.com/cloudsonic/sonic-server/itunesbridge" "github.com/google/wire" ) var allProviders = wire.NewSet( - itunesbridge.NewItunesControl, NewSystemController, NewBrowsingController, NewAlbumListController, diff --git a/conf/configuration.go b/conf/configuration.go index d6f703c51..a59f5de4b 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -29,7 +29,6 @@ type sonic struct { DevDisableAuthentication bool `default:"false"` DevDisableFileCheck bool `default:"false"` DevDisableBanner bool `default:"false"` - DevUseFileScanner bool `default:"false"` } var Sonic *sonic diff --git a/engine/browser.go b/engine/browser.go index abdda35c7..60a096f67 100644 --- a/engine/browser.go +++ b/engine/browser.go @@ -23,26 +23,20 @@ type Browser interface { GetGenres() (model.Genres, error) } -func NewBrowser(pr model.PropertyRepository, fr model.MediaFolderRepository, - ar model.ArtistRepository, alr model.AlbumRepository, mr model.MediaFileRepository, gr model.GenreRepository) Browser { - return &browser{pr, fr, ar, alr, mr, gr} +func NewBrowser(ds model.DataStore) Browser { + return &browser{ds} } type browser struct { - propRepo model.PropertyRepository - folderRepo model.MediaFolderRepository - artistRepo model.ArtistRepository - albumRepo model.AlbumRepository - mfileRepo model.MediaFileRepository - genreRepo model.GenreRepository + ds model.DataStore } func (b *browser) MediaFolders() (model.MediaFolders, error) { - return b.folderRepo.GetAll() + return b.ds.MediaFolder().GetAll() } func (b *browser) Indexes(ifModifiedSince time.Time) (model.ArtistIndexes, time.Time, error) { - l, err := b.propRepo.DefaultGet(model.PropLastScan, "-1") + l, err := b.ds.Property().DefaultGet(model.PropLastScan, "-1") ms, _ := strconv.ParseInt(l, 10, 64) lastModified := utils.ToTime(ms) @@ -51,7 +45,7 @@ func (b *browser) Indexes(ifModifiedSince time.Time) (model.ArtistIndexes, time. } if lastModified.After(ifModifiedSince) { - indexes, err := b.artistRepo.GetIndex() + indexes, err := b.ds.Artist().GetIndex() return indexes, lastModified, err } @@ -108,7 +102,7 @@ func (b *browser) Directory(ctx context.Context, id string) (*DirectoryInfo, err } func (b *browser) GetSong(id string) (*Entry, error) { - mf, err := b.mfileRepo.Get(id) + mf, err := b.ds.MediaFile().Get(id) if err != nil { return nil, err } @@ -118,7 +112,7 @@ func (b *browser) GetSong(id string) (*Entry, error) { } func (b *browser) GetGenres() (model.Genres, error) { - genres, err := b.genreRepo.GetAll() + genres, err := b.ds.Genre().GetAll() for i, g := range genres { if strings.TrimSpace(g.Name) == "" { genres[i].Name = "<Empty>" @@ -171,7 +165,7 @@ func (b *browser) buildAlbumDir(al *model.Album, tracks model.MediaFiles) *Direc } func (b *browser) isArtist(ctx context.Context, id string) bool { - found, err := b.artistRepo.Exists(id) + found, err := b.ds.Artist().Exists(id) if err != nil { log.Debug(ctx, "Error searching for Artist", "id", id, err) return false @@ -180,7 +174,7 @@ func (b *browser) isArtist(ctx context.Context, id string) bool { } func (b *browser) isAlbum(ctx context.Context, id string) bool { - found, err := b.albumRepo.Exists(id) + found, err := b.ds.Album().Exists(id) if err != nil { log.Debug(ctx, "Error searching for Album", "id", id, err) return false @@ -189,26 +183,26 @@ func (b *browser) isAlbum(ctx context.Context, id string) bool { } func (b *browser) retrieveArtist(id string) (a *model.Artist, as model.Albums, err error) { - a, err = b.artistRepo.Get(id) + a, err = b.ds.Artist().Get(id) if err != nil { err = fmt.Errorf("Error reading Artist %s from DB: %v", id, err) return } - if as, err = b.albumRepo.FindByArtist(id); err != nil { + if as, err = b.ds.Album().FindByArtist(id); err != nil { err = fmt.Errorf("Error reading %s's albums from DB: %v", a.Name, err) } return } func (b *browser) retrieveAlbum(id string) (al *model.Album, mfs model.MediaFiles, err error) { - al, err = b.albumRepo.Get(id) + al, err = b.ds.Album().Get(id) if err != nil { err = fmt.Errorf("Error reading Album %s from DB: %v", id, err) return } - if mfs, err = b.mfileRepo.FindByAlbum(id); err != nil { + if mfs, err = b.ds.MediaFile().FindByAlbum(id); err != nil { err = fmt.Errorf("Error reading %s's tracks from DB: %v", al.Name, err) } return diff --git a/engine/browser_test.go b/engine/browser_test.go index 30e1b8047..237579098 100644 --- a/engine/browser_test.go +++ b/engine/browser_test.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/cloudsonic/sonic-server/model" + "github.com/cloudsonic/sonic-server/persistence" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -18,7 +19,8 @@ var _ = Describe("Browser", func() { {Name: "", SongCount: 13, AlbumCount: 13}, {Name: "Electronic", SongCount: 4000, AlbumCount: 40}, }} - b = &browser{genreRepo: repo} + var ds = &persistence.MockDataStore{MockedGenre: repo} + b = &browser{ds: ds} }) It("returns sorted data", func() { diff --git a/engine/cover.go b/engine/cover.go index f35e7852c..9c6c04a5a 100644 --- a/engine/cover.go +++ b/engine/cover.go @@ -20,25 +20,24 @@ type Cover interface { } type cover struct { - mfileRepo model.MediaFileRepository - albumRepo model.AlbumRepository + ds model.DataStore } -func NewCover(mr model.MediaFileRepository, alr model.AlbumRepository) Cover { - return &cover{mr, alr} +func NewCover(ds model.DataStore) Cover { + return &cover{ds} } func (c *cover) getCoverPath(id string) (string, error) { switch { case strings.HasPrefix(id, "al-"): id = id[3:] - al, err := c.albumRepo.Get(id) + al, err := c.ds.Album().Get(id) if err != nil { return "", err } return al.CoverArtPath, nil default: - mf, err := c.mfileRepo.Get(id) + mf, err := c.ds.MediaFile().Get(id) if err != nil { return "", err } diff --git a/engine/cover_test.go b/engine/cover_test.go index 552c3cc29..2a1d39dbd 100644 --- a/engine/cover_test.go +++ b/engine/cover_test.go @@ -15,10 +15,11 @@ import ( func TestCover(t *testing.T) { Init(t, false) - mockMediaFileRepo := persistence.CreateMockMediaFileRepo() - mockAlbumRepo := persistence.CreateMockAlbumRepo() + ds := &persistence.MockDataStore{} + mockMediaFileRepo := ds.MediaFile().(*persistence.MockMediaFile) + mockAlbumRepo := ds.Album().(*persistence.MockAlbum) - cover := engine.NewCover(mockMediaFileRepo, mockAlbumRepo) + cover := engine.NewCover(ds) out := new(bytes.Buffer) Convey("Subject: GetCoverArt Endpoint", t, func() { diff --git a/engine/list_generator.go b/engine/list_generator.go index d4b81f0b4..8937778a0 100644 --- a/engine/list_generator.go +++ b/engine/list_generator.go @@ -22,22 +22,20 @@ type ListGenerator interface { GetRandomSongs(size int) (Entries, error) } -func NewListGenerator(arr model.ArtistRepository, alr model.AlbumRepository, mfr model.MediaFileRepository, npr NowPlayingRepository) ListGenerator { - return &listGenerator{arr, alr, mfr, npr} +func NewListGenerator(ds model.DataStore, npRepo NowPlayingRepository) ListGenerator { + return &listGenerator{ds, npRepo} } type listGenerator struct { - artistRepo model.ArtistRepository - albumRepo model.AlbumRepository - mfRepository model.MediaFileRepository - npRepo NowPlayingRepository + ds model.DataStore + npRepo NowPlayingRepository } // TODO: Only return albums that have the SortBy field != empty func (g *listGenerator) query(qo model.QueryOptions, offset int, size int) (Entries, error) { qo.Offset = offset qo.Size = size - albums, err := g.albumRepo.GetAll(qo) + albums, err := g.ds.Album().GetAll(qo) return FromAlbums(albums), err } @@ -73,7 +71,7 @@ func (g *listGenerator) GetByArtist(offset int, size int) (Entries, error) { } func (g *listGenerator) GetRandom(offset int, size int) (Entries, error) { - ids, err := g.albumRepo.GetAllIds() + ids, err := g.ds.Album().GetAllIds() if err != nil { return nil, err } @@ -83,7 +81,7 @@ func (g *listGenerator) GetRandom(offset int, size int) (Entries, error) { for i := 0; i < size; i++ { v := perm[i] - al, err := g.albumRepo.Get((ids)[v]) + al, err := g.ds.Album().Get((ids)[v]) if err != nil { return nil, err } @@ -93,7 +91,7 @@ func (g *listGenerator) GetRandom(offset int, size int) (Entries, error) { } func (g *listGenerator) GetRandomSongs(size int) (Entries, error) { - ids, err := g.mfRepository.GetAllIds() + ids, err := g.ds.MediaFile().GetAllIds() if err != nil { return nil, err } @@ -103,7 +101,7 @@ func (g *listGenerator) GetRandomSongs(size int) (Entries, error) { for i := 0; i < size; i++ { v := perm[i] - mf, err := g.mfRepository.Get(ids[v]) + mf, err := g.ds.MediaFile().Get(ids[v]) if err != nil { return nil, err } @@ -114,7 +112,7 @@ func (g *listGenerator) GetRandomSongs(size int) (Entries, error) { func (g *listGenerator) GetStarred(offset int, size int) (Entries, error) { qo := model.QueryOptions{Offset: offset, Size: size, SortBy: "starred_at", Desc: true} - albums, err := g.albumRepo.GetStarred(qo) + albums, err := g.ds.Album().GetStarred(qo) if err != nil { return nil, err } @@ -124,7 +122,7 @@ func (g *listGenerator) GetStarred(offset int, size int) (Entries, error) { // TODO Return is confusing func (g *listGenerator) GetAllStarred() (Entries, Entries, Entries, error) { - artists, err := g.artistRepo.GetStarred(model.QueryOptions{SortBy: "starred_at", Desc: true}) + artists, err := g.ds.Artist().GetStarred(model.QueryOptions{SortBy: "starred_at", Desc: true}) if err != nil { return nil, nil, nil, err } @@ -134,7 +132,7 @@ func (g *listGenerator) GetAllStarred() (Entries, Entries, Entries, error) { return nil, nil, nil, err } - mediaFiles, err := g.mfRepository.GetStarred(model.QueryOptions{SortBy: "starred_at", Desc: true}) + mediaFiles, err := g.ds.MediaFile().GetStarred(model.QueryOptions{SortBy: "starred_at", Desc: true}) if err != nil { return nil, nil, nil, err } @@ -149,7 +147,7 @@ func (g *listGenerator) GetNowPlaying() (Entries, error) { } entries := make(Entries, len(npInfo)) for i, np := range npInfo { - mf, err := g.mfRepository.Get(np.TrackID) + mf, err := g.ds.MediaFile().Get(np.TrackID) if err != nil { return nil, err } diff --git a/engine/playlists.go b/engine/playlists.go index b56ef4b90..bf8d3280d 100644 --- a/engine/playlists.go +++ b/engine/playlists.go @@ -2,10 +2,7 @@ package engine import ( "context" - "sort" - "github.com/cloudsonic/sonic-server/itunesbridge" - "github.com/cloudsonic/sonic-server/log" "github.com/cloudsonic/sonic-server/model" ) @@ -17,18 +14,16 @@ type Playlists interface { Update(playlistId string, name *string, idsToAdd []string, idxToRemove []int) error } -func NewPlaylists(itunes itunesbridge.ItunesControl, pr model.PlaylistRepository, mr model.MediaFileRepository) Playlists { - return &playlists{itunes, pr, mr} +func NewPlaylists(ds model.DataStore) Playlists { + return &playlists{ds} } type playlists struct { - itunes itunesbridge.ItunesControl - plsRepo model.PlaylistRepository - mfileRepo model.MediaFileRepository + ds model.DataStore } func (p *playlists) GetAll() (model.Playlists, error) { - return p.plsRepo.GetAll(model.QueryOptions{}) + return p.ds.Playlist().GetAll(model.QueryOptions{}) } type PlaylistInfo struct { @@ -43,52 +38,22 @@ type PlaylistInfo struct { } func (p *playlists) Create(ctx context.Context, name string, ids []string) error { - pid, err := p.itunes.CreatePlaylist(name, ids) - if err != nil { - return err - } - log.Info(ctx, "Created playlist", "playlist", name, "id", pid) + // TODO return nil } func (p *playlists) Delete(ctx context.Context, playlistId string) error { - err := p.itunes.DeletePlaylist(playlistId) - if err != nil { - return err - } - log.Info(ctx, "Deleted playlist", "id", playlistId) + // TODO return nil } func (p *playlists) Update(playlistId string, name *string, idsToAdd []string, idxToRemove []int) error { - pl, err := p.plsRepo.Get(playlistId) - if err != nil { - return err - } - if name != nil { - pl.Name = *name - err := p.itunes.RenamePlaylist(pl.ID, pl.Name) - if err != nil { - return err - } - } - if len(idsToAdd) > 0 || len(idxToRemove) > 0 { - sort.Sort(sort.Reverse(sort.IntSlice(idxToRemove))) - for _, i := range idxToRemove { - pl.Tracks, pl.Tracks[len(pl.Tracks)-1] = append(pl.Tracks[:i], pl.Tracks[i+1:]...), "" - } - pl.Tracks = append(pl.Tracks, idsToAdd...) - err := p.itunes.UpdatePlaylist(pl.ID, pl.Tracks) - if err != nil { - return err - } - } - p.plsRepo.Put(pl) // Ignores errors, as any changes will be overridden in the next scan + // TODO return nil } func (p *playlists) Get(id string) (*PlaylistInfo, error) { - pl, err := p.plsRepo.Get(id) + pl, err := p.ds.Playlist().Get(id) if err != nil { return nil, err } @@ -96,7 +61,7 @@ func (p *playlists) Get(id string) (*PlaylistInfo, error) { pinfo := &PlaylistInfo{ Id: pl.ID, Name: pl.Name, - SongCount: len(pl.Tracks), + SongCount: len(pl.Tracks), // TODO Use model.Playlist Duration: pl.Duration, Public: pl.Public, Owner: pl.Owner, @@ -106,7 +71,7 @@ func (p *playlists) Get(id string) (*PlaylistInfo, error) { // TODO Optimize: Get all tracks at once for i, mfId := range pl.Tracks { - mf, err := p.mfileRepo.Get(mfId) + mf, err := p.ds.MediaFile().Get(mfId) if err != nil { return nil, err } diff --git a/engine/ratings.go b/engine/ratings.go index 2018cffed..d2cf58122 100644 --- a/engine/ratings.go +++ b/engine/ratings.go @@ -3,11 +3,7 @@ package engine import ( "context" - "github.com/cloudsonic/sonic-server/conf" - "github.com/cloudsonic/sonic-server/itunesbridge" - "github.com/cloudsonic/sonic-server/log" "github.com/cloudsonic/sonic-server/model" - "github.com/cloudsonic/sonic-server/utils" ) type Ratings interface { @@ -15,86 +11,30 @@ type Ratings interface { SetRating(ctx context.Context, id string, rating int) error } -func NewRatings(itunes itunesbridge.ItunesControl, mr model.MediaFileRepository, alr model.AlbumRepository, ar model.ArtistRepository) Ratings { - return &ratings{itunes, mr, alr, ar} +func NewRatings(ds model.DataStore) Ratings { + return &ratings{ds} } type ratings struct { - itunes itunesbridge.ItunesControl - mfRepo model.MediaFileRepository - albumRepo model.AlbumRepository - artistRepo model.ArtistRepository + ds model.DataStore } func (r ratings) SetRating(ctx context.Context, id string, rating int) error { - rating = utils.MinInt(rating, 5) * 20 - - isAlbum, _ := r.albumRepo.Exists(id) - if isAlbum { - mfs, _ := r.mfRepo.FindByAlbum(id) - if len(mfs) > 0 { - log.Debug(ctx, "Set Rating", "value", rating, "album", mfs[0].Album) - if err := r.itunes.SetAlbumRating(mfs[0].ID, rating); err != nil { - return err - } - } - return nil - } - - mf, err := r.mfRepo.Get(id) - if err != nil { - return err - } - if mf != nil { - log.Debug(ctx, "Set Rating", "value", rating, "song", mf.Title) - if err := r.itunes.SetTrackRating(mf.ID, rating); err != nil { - return err - } - return nil - } + // TODO return model.ErrNotFound } func (r ratings) SetStar(ctx context.Context, star bool, ids ...string) error { - if conf.Sonic.DevUseFileScanner { - err := r.mfRepo.SetStar(star, ids...) + return r.ds.WithTx(func(tx model.DataStore) error { + err := tx.MediaFile().SetStar(star, ids...) if err != nil { return err } - err = r.albumRepo.SetStar(star, ids...) + err = tx.Album().SetStar(star, ids...) if err != nil { return err } - err = r.artistRepo.SetStar(star, ids...) + err = tx.Artist().SetStar(star, ids...) return err - } - - for _, id := range ids { - isAlbum, _ := r.albumRepo.Exists(id) - if isAlbum { - mfs, _ := r.mfRepo.FindByAlbum(id) - if len(mfs) > 0 { - log.Debug(ctx, "Set Star", "value", star, "album", mfs[0].Album) - if err := r.itunes.SetAlbumLoved(mfs[0].ID, star); err != nil { - return err - } - } - continue - } - - mf, err := r.mfRepo.Get(id) - if err != nil { - return err - } - if mf != nil { - log.Debug(ctx, "Set Star", "value", star, "song", mf.Title) - if err := r.itunes.SetTrackLoved(mf.ID, star); err != nil { - return err - } - continue - } - return model.ErrNotFound - } - - return nil + }) } diff --git a/engine/scrobbler.go b/engine/scrobbler.go index b9cac14f0..742a33382 100644 --- a/engine/scrobbler.go +++ b/engine/scrobbler.go @@ -6,9 +6,6 @@ import ( "fmt" "time" - "github.com/cloudsonic/sonic-server/conf" - "github.com/cloudsonic/sonic-server/itunesbridge" - "github.com/cloudsonic/sonic-server/log" "github.com/cloudsonic/sonic-server/model" ) @@ -22,87 +19,31 @@ type Scrobbler interface { NowPlaying(ctx context.Context, playerId int, playerName, trackId, username string) (*model.MediaFile, error) } -func NewScrobbler(itunes itunesbridge.ItunesControl, mr model.MediaFileRepository, alr model.AlbumRepository, npr NowPlayingRepository) Scrobbler { - return &scrobbler{itunes: itunes, mfRepo: mr, alRepo: alr, npRepo: npr} +func NewScrobbler(ds model.DataStore, npr NowPlayingRepository) Scrobbler { + return &scrobbler{ds: ds, npRepo: npr} } type scrobbler struct { - itunes itunesbridge.ItunesControl - mfRepo model.MediaFileRepository - alRepo model.AlbumRepository + ds model.DataStore npRepo NowPlayingRepository } -func (s *scrobbler) detectSkipped(ctx context.Context, playerId int, trackId string) { - size, _ := s.npRepo.Count(playerId) - switch size { - case 0: - return - case 1: - np, _ := s.npRepo.Tail(playerId) - if np.TrackID != trackId { - return - } - s.npRepo.Dequeue(playerId) - default: - prev, _ := s.npRepo.Dequeue(playerId) - for { - if prev.TrackID == trackId { - break - } - np, err := s.npRepo.Dequeue(playerId) - if np == nil || err != nil { - break - } - diff := np.Start.Sub(prev.Start) - if diff < minSkipped || diff > maxSkipped { - log.Debug(ctx, fmt.Sprintf("-- Playtime for track %s was %v. Not skipping.", prev.TrackID, diff)) - prev = np - continue - } - err = s.itunes.MarkAsSkipped(prev.TrackID, prev.Start.Add(1*time.Minute)) - if err != nil { - log.Warn(ctx, "Error skipping track", "id", prev.TrackID) - } else { - log.Debug(ctx, "-- Skipped track "+prev.TrackID) - } - } - } -} - func (s *scrobbler) Register(ctx context.Context, playerId int, trackId string, playTime time.Time) (*model.MediaFile, error) { - s.detectSkipped(ctx, playerId, trackId) - - if conf.Sonic.DevUseFileScanner { - mf, err := s.mfRepo.Get(trackId) - if err != nil { - return nil, err - } - err = s.mfRepo.MarkAsPlayed(trackId, playTime) - if err != nil { - return nil, err - } - err = s.alRepo.MarkAsPlayed(mf.AlbumID, playTime) - return mf, err - } - - mf, err := s.mfRepo.Get(trackId) + // TODO Add transaction + mf, err := s.ds.MediaFile().Get(trackId) if err != nil { return nil, err } - - if mf == nil { - return nil, errors.New(fmt.Sprintf(`ID "%s" not found`, trackId)) - } - - if err := s.itunes.MarkAsPlayed(trackId, playTime); err != nil { + err = s.ds.MediaFile().MarkAsPlayed(trackId, playTime) + if err != nil { return nil, err } - return mf, nil + err = s.ds.Album().MarkAsPlayed(mf.AlbumID, playTime) + return mf, err } func (s *scrobbler) NowPlaying(ctx context.Context, playerId int, playerName, trackId, username string) (*model.MediaFile, error) { - mf, err := s.mfRepo.Get(trackId) + mf, err := s.ds.MediaFile().Get(trackId) if err != nil { return nil, err } diff --git a/engine/scrobbler_test.go b/engine/scrobbler_test.go deleted file mode 100644 index eb000e3bf..000000000 --- a/engine/scrobbler_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package engine_test - -import ( - "errors" - "testing" - "time" - - "github.com/cloudsonic/sonic-server/engine" - "github.com/cloudsonic/sonic-server/itunesbridge" - "github.com/cloudsonic/sonic-server/persistence" - . "github.com/cloudsonic/sonic-server/tests" - . "github.com/smartystreets/goconvey/convey" -) - -func TestScrobbler(t *testing.T) { - - Init(t, false) - - mfRepo := persistence.CreateMockMediaFileRepo() - alRepo := persistence.CreateMockAlbumRepo() - npRepo := engine.CreateMockNowPlayingRepo() - itCtrl := &mockItunesControl{} - - scrobbler := engine.NewScrobbler(itCtrl, mfRepo, alRepo, npRepo) - - Convey("Given a DB with one song", t, func() { - mfRepo.SetData(`[{"ID":"2","Title":"Hands Of Time"}]`, 1) - - Convey("When I scrobble an existing song", func() { - now := time.Now() - mf, err := scrobbler.Register(nil, 1, "2", now) - - Convey("Then I get the scrobbled song back", func() { - So(err, ShouldBeNil) - So(mf.Title, ShouldEqual, "Hands Of Time") - }) - - Convey("And iTunes is notified", func() { - So(itCtrl.played, ShouldContainKey, "2") - So(itCtrl.played["2"].Equal(now), ShouldBeTrue) - }) - - }) - - Convey("When the ID is not in the DB", func() { - _, err := scrobbler.Register(nil, 1, "3", time.Now()) - - Convey("Then I receive an error", func() { - So(err, ShouldNotBeNil) - }) - - Convey("And iTunes is not notified", func() { - So(itCtrl.played, ShouldNotContainKey, "3") - }) - }) - - Convey("When I inform the song that is now playing", func() { - mf, err := scrobbler.NowPlaying(nil, 1, "DSub", "2", "deluan") - - Convey("Then I get the song for that id back", func() { - So(err, ShouldBeNil) - So(mf.Title, ShouldEqual, "Hands Of Time") - }) - - Convey("And it saves the song as the one current playing", func() { - info, _ := npRepo.Head(1) - So(info.TrackID, ShouldEqual, "2") - // Commenting out time sensitive test, due to flakiness - // So(info.Start, ShouldHappenBefore, time.Now()) - So(info.Username, ShouldEqual, "deluan") - So(info.PlayerName, ShouldEqual, "DSub") - }) - - Convey("And iTunes is not notified", func() { - So(itCtrl.played, ShouldNotContainKey, "2") - }) - }) - - Reset(func() { - itCtrl.played = make(map[string]time.Time) - itCtrl.skipped = make(map[string]time.Time) - }) - - }) -} - -var aPointInTime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) - -func TestSkipping(t *testing.T) { - Init(t, false) - - mfRepo := persistence.CreateMockMediaFileRepo() - alRepo := persistence.CreateMockAlbumRepo() - npRepo := engine.CreateMockNowPlayingRepo() - itCtrl := &mockItunesControl{} - - scrobbler := engine.NewScrobbler(itCtrl, mfRepo, alRepo, npRepo) - - Convey("Given a DB with three songs", t, func() { - mfRepo.SetData(`[{"ID":"1","Title":"Femme Fatale"},{"ID":"2","Title":"Here She Comes Now"},{"ID":"3","Title":"Lady Godiva's Operation"}]`, 3) - itCtrl.skipped = make(map[string]time.Time) - npRepo.ClearAll() - Convey("When I skip 2 songs", func() { - npRepo.OverrideNow(aPointInTime) - scrobbler.NowPlaying(nil, 1, "DSub", "1", "deluan") - npRepo.OverrideNow(aPointInTime.Add(2 * time.Second)) - scrobbler.NowPlaying(nil, 1, "DSub", "3", "deluan") - npRepo.OverrideNow(aPointInTime.Add(3 * time.Second)) - scrobbler.NowPlaying(nil, 1, "DSub", "2", "deluan") - Convey("Then the NowPlaying song should be the last one", func() { - np, err := npRepo.GetAll() - So(err, ShouldBeNil) - So(np, ShouldHaveLength, 1) - So(np[0].TrackID, ShouldEqual, "2") - }) - }) - Convey("When I play one song", func() { - npRepo.OverrideNow(aPointInTime) - scrobbler.NowPlaying(nil, 1, "DSub", "1", "deluan") - Convey("And I skip it before 20 seconds", func() { - npRepo.OverrideNow(aPointInTime.Add(7 * time.Second)) - scrobbler.NowPlaying(nil, 1, "DSub", "2", "deluan") - Convey("Then the first song should be marked as skipped", func() { - mf, err := scrobbler.Register(nil, 1, "2", aPointInTime.Add(3*time.Minute)) - So(mf.ID, ShouldEqual, "2") - So(itCtrl.skipped, ShouldContainKey, "1") - So(err, ShouldBeNil) - }) - }) - Convey("And I skip it before 3 seconds", func() { - npRepo.OverrideNow(aPointInTime.Add(2 * time.Second)) - scrobbler.NowPlaying(nil, 1, "DSub", "2", "deluan") - Convey("Then the first song should be marked as skipped", func() { - mf, err := scrobbler.Register(nil, 1, "2", aPointInTime.Add(3*time.Minute)) - So(mf.ID, ShouldEqual, "2") - So(itCtrl.skipped, ShouldBeEmpty) - So(err, ShouldBeNil) - }) - }) - Convey("And I skip it after 20 seconds", func() { - npRepo.OverrideNow(aPointInTime.Add(30 * time.Second)) - scrobbler.NowPlaying(nil, 1, "DSub", "2", "deluan") - Convey("Then the first song should be marked as skipped", func() { - mf, err := scrobbler.Register(nil, 1, "2", aPointInTime.Add(3*time.Minute)) - So(mf.ID, ShouldEqual, "2") - So(itCtrl.skipped, ShouldBeEmpty) - So(err, ShouldBeNil) - }) - }) - Convey("And I scrobble it before starting to play the other song", func() { - mf, err := scrobbler.Register(nil, 1, "1", time.Now()) - Convey("Then the first song should NOT marked as skipped", func() { - So(mf.ID, ShouldEqual, "1") - So(itCtrl.skipped, ShouldBeEmpty) - So(err, ShouldBeNil) - }) - }) - }) - Convey("When the NowPlaying for the next song happens before the Scrobble", func() { - npRepo.OverrideNow(aPointInTime) - scrobbler.NowPlaying(nil, 1, "DSub", "1", "deluan") - npRepo.OverrideNow(aPointInTime.Add(10 * time.Second)) - scrobbler.NowPlaying(nil, 1, "DSub", "2", "deluan") - scrobbler.Register(nil, 1, "1", aPointInTime.Add(10*time.Minute)) - Convey("Then the NowPlaying song should be the last one", func() { - np, _ := npRepo.GetAll() - So(np, ShouldHaveLength, 1) - So(np[0].TrackID, ShouldEqual, "2") - }) - }) - }) -} - -type mockItunesControl struct { - itunesbridge.ItunesControl - played map[string]time.Time - skipped map[string]time.Time - error bool -} - -func (m *mockItunesControl) MarkAsPlayed(id string, playDate time.Time) error { - if m.error { - return errors.New("ID not found") - } - if m.played == nil { - m.played = make(map[string]time.Time) - } - m.played[id] = playDate - return nil -} - -func (m *mockItunesControl) MarkAsSkipped(id string, skipDate time.Time) error { - if m.error { - return errors.New("ID not found") - } - if m.skipped == nil { - m.skipped = make(map[string]time.Time) - } - m.skipped[id] = skipDate - return nil -} diff --git a/engine/search.go b/engine/search.go index d52c20f02..295abf118 100644 --- a/engine/search.go +++ b/engine/search.go @@ -15,19 +15,17 @@ type Search interface { } type search struct { - artistRepo model.ArtistRepository - albumRepo model.AlbumRepository - mfileRepo model.MediaFileRepository + ds model.DataStore } -func NewSearch(ar model.ArtistRepository, alr model.AlbumRepository, mr model.MediaFileRepository) Search { - s := &search{artistRepo: ar, albumRepo: alr, mfileRepo: mr} +func NewSearch(ds model.DataStore) Search { + s := &search{ds} return s } func (s *search) SearchArtist(ctx context.Context, q string, offset int, size int) (Entries, error) { q = sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))) - resp, err := s.artistRepo.Search(q, offset, size) + resp, err := s.ds.Artist().Search(q, offset, size) if err != nil { return nil, nil } @@ -40,7 +38,7 @@ func (s *search) SearchArtist(ctx context.Context, q string, offset int, size in func (s *search) SearchAlbum(ctx context.Context, q string, offset int, size int) (Entries, error) { q = sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))) - resp, err := s.albumRepo.Search(q, offset, size) + resp, err := s.ds.Album().Search(q, offset, size) if err != nil { return nil, nil } @@ -53,7 +51,7 @@ func (s *search) SearchAlbum(ctx context.Context, q string, offset int, size int func (s *search) SearchSong(ctx context.Context, q string, offset int, size int) (Entries, error) { q = sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))) - resp, err := s.mfileRepo.Search(q, offset, size) + resp, err := s.ds.MediaFile().Search(q, offset, size) if err != nil { return nil, nil } diff --git a/itunesbridge/itunes.go b/itunesbridge/itunes.go deleted file mode 100644 index a2ec391b5..000000000 --- a/itunesbridge/itunes.go +++ /dev/null @@ -1,135 +0,0 @@ -package itunesbridge - -import ( - "fmt" - "strings" - "time" -) - -type ItunesControl interface { - MarkAsPlayed(trackId string, playDate time.Time) error - MarkAsSkipped(trackId string, skipDate time.Time) error - SetTrackLoved(trackId string, loved bool) error - SetAlbumLoved(trackId string, loved bool) error - SetTrackRating(trackId string, rating int) error - SetAlbumRating(trackId string, rating int) error - CreatePlaylist(name string, ids []string) (string, error) - UpdatePlaylist(playlistId string, ids []string) error - RenamePlaylist(playlistId, name string) error - DeletePlaylist(playlistId string) error -} - -func NewItunesControl() ItunesControl { - return &itunesControl{} -} - -type itunesControl struct{} - -func (c *itunesControl) CreatePlaylist(name string, ids []string) (string, error) { - pids := `"` + strings.Join(ids, `","`) + `"` - script := Script{ - fmt.Sprintf(`set pls to (make new user playlist with properties {name:"%s"})`, name), - fmt.Sprintf(`set pids to {%s}`, pids), - `repeat with trackPID in pids`, - ` set myTrack to the first item of (every track whose persistent ID is equal to trackPID)`, - ` duplicate myTrack to pls`, - `end repeat`, - `persistent ID of pls`} - pid, err := script.OutputString() - if err != nil { - return "", err - } - return strings.TrimSuffix(pid, "\n"), nil -} - -func (c *itunesControl) UpdatePlaylist(playlistId string, ids []string) error { - pids := `"` + strings.Join(ids, `","`) + `"` - script := Script{ - fmt.Sprintf(`set pls to the first item of (every playlist whose persistent ID is equal to "%s")`, playlistId), - `delete every track of pls`, - fmt.Sprintf(`set pids to {%s}`, pids), - `repeat with trackPID in pids`, - ` set myTrack to the first item of (every track whose persistent ID is equal to trackPID)`, - ` duplicate myTrack to pls`, - `end repeat`} - return script.Run() -} - -func (c *itunesControl) RenamePlaylist(playlistId, name string) error { - script := Script{ - fmt.Sprintf(`set pls to the first item of (every playlist whose persistent ID is equal to "%s")`, playlistId), - `tell pls`, - fmt.Sprintf(`set name to "%s"`, name), - `end tell`} - return script.Run() -} - -func (c *itunesControl) DeletePlaylist(playlistId string) error { - script := Script{ - fmt.Sprintf(`set pls to the first item of (every playlist whose persistent ID is equal to "%s")`, playlistId), - `delete pls`, - } - return script.Run() -} - -func (c *itunesControl) MarkAsPlayed(trackId string, playDate time.Time) error { - script := Script{fmt.Sprintf( - `set theTrack to the first item of (every track whose persistent ID is equal to "%s")`, trackId), - `set c to (get played count of theTrack)`, - `tell theTrack`, - `set played count to c + 1`, - fmt.Sprintf(`set played date to date("%s")`, c.formatDateTime(playDate)), - `end tell`} - return script.Run() -} - -func (c *itunesControl) MarkAsSkipped(trackId string, skipDate time.Time) error { - script := Script{fmt.Sprintf( - `set theTrack to the first item of (every track whose persistent ID is equal to "%s")`, trackId), - `set c to (get skipped count of theTrack)`, - `tell theTrack`, - `set skipped count to c + 1`, - fmt.Sprintf(`set skipped date to date("%s")`, c.formatDateTime(skipDate)), - `end tell`} - return script.Run() -} - -func (c *itunesControl) SetTrackLoved(trackId string, loved bool) error { - script := Script{fmt.Sprintf( - `set theTrack to the first item of (every track whose persistent ID is equal to "%s")`, trackId), - `tell theTrack`, - fmt.Sprintf(`set loved to %v`, loved), - `end tell`} - return script.Run() -} - -func (c *itunesControl) SetAlbumLoved(trackId string, loved bool) error { - script := Script{fmt.Sprintf( - `set theTrack to the first item of (every track whose persistent ID is equal to "%s")`, trackId), - `tell theTrack`, - fmt.Sprintf(`set album loved to %v`, loved), - `end tell`} - return script.Run() -} - -func (c *itunesControl) SetTrackRating(trackId string, rating int) error { - script := Script{fmt.Sprintf( - `set theTrack to the first item of (every track whose persistent ID is equal to "%s")`, trackId), - `tell theTrack`, - fmt.Sprintf(`set rating to %d`, rating), - `end tell`} - return script.Run() -} - -func (c *itunesControl) SetAlbumRating(trackId string, rating int) error { - script := Script{fmt.Sprintf( - `set theTrack to the first item of (every track whose persistent ID is equal to "%s")`, trackId), - `tell theTrack`, - fmt.Sprintf(`set album rating to %d`, rating), - `end tell`} - return script.Run() -} - -func (c *itunesControl) formatDateTime(d time.Time) string { - return d.Format("Jan _2, 2006 3:04PM") -} diff --git a/itunesbridge/script.go b/itunesbridge/script.go deleted file mode 100644 index c55b8a00e..000000000 --- a/itunesbridge/script.go +++ /dev/null @@ -1,63 +0,0 @@ -package itunesbridge - -import ( - "fmt" - "io" - "os" - "os/exec" -) - -// Original from https://github.com/bmatsuo/tuner -type Script []string - -var CommandHost string - -func (s Script) lines() []string { - if len(s) == 0 { - panic("empty script") - } - - lines := make([]string, 0, 2) - tell := `tell application "iTunes"` - if CommandHost != "" { - tell += fmt.Sprintf(` of machine %q`, CommandHost) - } - if len(s) == 1 { - tell += " to " + s[0] - lines = append(lines, tell) - } else { - lines = append(lines, tell) - lines = append(lines, s...) - lines = append(lines, "end tell") - } - return lines -} - -func (s Script) args() []string { - var args []string - for _, line := range s.lines() { - args = append(args, "-e", line) - } - return args -} - -func (s Script) Command(w io.Writer, args ...string) *exec.Cmd { - command := exec.Command("osascript", append(s.args(), args...)...) - command.Stdout = w - command.Stderr = os.Stderr - return command -} - -func (s Script) Run(args ...string) error { - return s.Command(os.Stdout, args...).Run() -} - -func (s Script) Output(args ...string) ([]byte, error) { - return s.Command(nil, args...).Output() -} - -func (s Script) OutputString(args ...string) (string, error) { - p, err := s.Output(args...) - str := string(p) - return str, err -} diff --git a/model/base.go b/model/model.go similarity index 62% rename from model/base.go rename to model/model.go index b804f7f3f..535027405 100644 --- a/model/base.go +++ b/model/model.go @@ -1,6 +1,8 @@ package model -import "errors" +import ( + "errors" +) var ( ErrNotFound = errors.New("data not found") @@ -19,3 +21,15 @@ type QueryOptions struct { Size int Filters Filters } + +type DataStore interface { + Album() AlbumRepository + Artist() ArtistRepository + MediaFile() MediaFileRepository + MediaFolder() MediaFolderRepository + Genre() GenreRepository + Playlist() PlaylistRepository + Property() PropertyRepository + + WithTx(func(tx DataStore) error) error +} diff --git a/persistence/album_repository.go b/persistence/album_repository.go index f51b581f6..defc2ba49 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -36,22 +36,21 @@ type albumRepository struct { searchableRepository } -func NewAlbumRepository() model.AlbumRepository { +func NewAlbumRepository(o orm.Ormer) model.AlbumRepository { r := &albumRepository{} + r.ormer = o r.tableName = "album" return r } func (r *albumRepository) Put(a *model.Album) error { ta := album(*a) - return withTx(func(o orm.Ormer) error { - return r.put(o, a.ID, a.Name, &ta) - }) + return r.put(a.ID, a.Name, &ta) } func (r *albumRepository) Get(id string) (*model.Album, error) { ta := album{ID: id} - err := Db().Read(&ta) + err := r.ormer.Read(&ta) if err == orm.ErrNoRows { return nil, model.ErrNotFound } @@ -64,7 +63,7 @@ func (r *albumRepository) Get(id string) (*model.Album, error) { func (r *albumRepository) FindByArtist(artistId string) (model.Albums, error) { var albums []album - _, err := r.newQuery(Db()).Filter("artist_id", artistId).OrderBy("year", "name").All(&albums) + _, err := r.newQuery().Filter("artist_id", artistId).OrderBy("year", "name").All(&albums) if err != nil { return nil, err } @@ -73,7 +72,7 @@ func (r *albumRepository) FindByArtist(artistId string) (model.Albums, error) { func (r *albumRepository) GetAll(options ...model.QueryOptions) (model.Albums, error) { var all []album - _, err := r.newQuery(Db(), options...).All(&all) + _, err := r.newQuery(options...).All(&all) if err != nil { return nil, err } @@ -95,7 +94,7 @@ func (r *albumRepository) Refresh(ids ...string) error { HasCoverArt bool } var albums []refreshAlbum - o := Db() + o := r.ormer sql := fmt.Sprintf(` select album_id as id, album as name, f.artist, f.album_artist, f.artist_id, f.compilation, f.genre, max(f.year) as year, sum(f.play_count) as play_count, max(f.play_date) as play_date, sum(f.duration) as duration, @@ -126,7 +125,7 @@ group by album_id order by f.id`, strings.Join(ids, "','")) } else { toInsert = append(toInsert, al.album) } - err := r.addToIndex(o, r.tableName, al.ID, al.Name) + err := r.addToIndex(r.tableName, al.ID, al.Name) if err != nil { return err } @@ -153,23 +152,20 @@ group by album_id order by f.id`, strings.Join(ids, "','")) } func (r *albumRepository) PurgeInactive(activeList model.Albums) error { - return withTx(func(o orm.Ormer) error { - _, err := r.purgeInactive(o, activeList, func(item interface{}) string { - return item.(model.Album).ID - }) - return err + _, err := r.purgeInactive(activeList, func(item interface{}) string { + return item.(model.Album).ID }) + return err } func (r *albumRepository) PurgeEmpty() error { - o := Db() - _, err := o.Raw("delete from album where id not in (select distinct(album_id) from media_file)").Exec() + _, err := r.ormer.Raw("delete from album where id not in (select distinct(album_id) from media_file)").Exec() return err } func (r *albumRepository) GetStarred(options ...model.QueryOptions) (model.Albums, error) { var starred []album - _, err := r.newQuery(Db(), options...).Filter("starred", true).All(&starred) + _, err := r.newQuery(options...).Filter("starred", true).All(&starred) if err != nil { return nil, err } @@ -184,7 +180,7 @@ func (r *albumRepository) SetStar(starred bool, ids ...string) error { if starred { starredAt = time.Now() } - _, err := r.newQuery(Db()).Filter("id__in", ids).Update(orm.Params{ + _, err := r.newQuery().Filter("id__in", ids).Update(orm.Params{ "starred": starred, "starred_at": starredAt, }) @@ -192,7 +188,7 @@ func (r *albumRepository) SetStar(starred bool, ids ...string) error { } func (r *albumRepository) MarkAsPlayed(id string, playDate time.Time) error { - _, err := r.newQuery(Db()).Filter("id", id).Update(orm.Params{ + _, err := r.newQuery().Filter("id", id).Update(orm.Params{ "play_count": orm.ColValue(orm.ColAdd, 1), "play_date": playDate, }) diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index b125c92b5..2624fdd70 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -1,6 +1,7 @@ package persistence import ( + "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/model" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -10,7 +11,7 @@ var _ = Describe("AlbumRepository", func() { var repo model.AlbumRepository BeforeEach(func() { - repo = NewAlbumRepository() + repo = NewAlbumRepository(orm.NewOrm()) }) Describe("GetAll", func() { diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index 077dc7385..f17e223ec 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -26,8 +26,9 @@ type artistRepository struct { indexGroups utils.IndexGroups } -func NewArtistRepository() model.ArtistRepository { +func NewArtistRepository(o orm.Ormer) model.ArtistRepository { r := &artistRepository{} + r.ormer = o r.indexGroups = utils.ParseIndexGroups(conf.Sonic.IndexGroups) r.tableName = "artist" return r @@ -46,14 +47,12 @@ func (r *artistRepository) getIndexKey(a *artist) string { func (r *artistRepository) Put(a *model.Artist) error { ta := artist(*a) - return withTx(func(o orm.Ormer) error { - return r.put(o, a.ID, a.Name, &ta) - }) + return r.put(a.ID, a.Name, &ta) } func (r *artistRepository) Get(id string) (*model.Artist, error) { ta := artist{ID: id} - err := Db().Read(&ta) + err := r.ormer.Read(&ta) if err == orm.ErrNoRows { return nil, model.ErrNotFound } @@ -68,7 +67,7 @@ func (r *artistRepository) Get(id string) (*model.Artist, error) { func (r *artistRepository) GetIndex() (model.ArtistIndexes, error) { var all []artist // TODO Paginate - _, err := r.newQuery(Db()).OrderBy("name").All(&all) + _, err := r.newQuery().OrderBy("name").All(&all) if err != nil { return nil, err } @@ -101,7 +100,7 @@ func (r *artistRepository) Refresh(ids ...string) error { Compilation bool } var artists []refreshArtist - o := Db() + o := r.ormer sql := fmt.Sprintf(` select f.artist_id as id, f.artist as name, @@ -131,7 +130,7 @@ where f.artist_id in ('%s') group by f.artist_id order by f.id`, strings.Join(id } else { toInsert = append(toInsert, ar.artist) } - err := r.addToIndex(o, r.tableName, ar.ID, ar.Name) + err := r.addToIndex(r.tableName, ar.ID, ar.Name) if err != nil { return err } @@ -158,7 +157,7 @@ where f.artist_id in ('%s') group by f.artist_id order by f.id`, strings.Join(id func (r *artistRepository) GetStarred(options ...model.QueryOptions) (model.Artists, error) { var starred []artist - _, err := r.newQuery(Db(), options...).Filter("starred", true).All(&starred) + _, err := r.newQuery(options...).Filter("starred", true).All(&starred) if err != nil { return nil, err } @@ -173,7 +172,7 @@ func (r *artistRepository) SetStar(starred bool, ids ...string) error { if starred { starredAt = time.Now() } - _, err := r.newQuery(Db()).Filter("id__in", ids).Update(orm.Params{ + _, err := r.newQuery().Filter("id__in", ids).Update(orm.Params{ "starred": starred, "starred_at": starredAt, }) @@ -181,17 +180,14 @@ func (r *artistRepository) SetStar(starred bool, ids ...string) error { } func (r *artistRepository) PurgeInactive(activeList model.Artists) error { - return withTx(func(o orm.Ormer) error { - _, err := r.purgeInactive(o, activeList, func(item interface{}) string { - return item.(model.Artist).ID - }) - return err + _, err := r.purgeInactive(activeList, func(item interface{}) string { + return item.(model.Artist).ID }) + return err } func (r *artistRepository) PurgeEmpty() error { - o := Db() - _, err := o.Raw("delete from artist where id not in (select distinct(artist_id) from album)").Exec() + _, err := r.ormer.Raw("delete from artist where id not in (select distinct(artist_id) from album)").Exec() return err } diff --git a/persistence/artist_repository_test.go b/persistence/artist_repository_test.go index 0bd9042df..60ed8d63f 100644 --- a/persistence/artist_repository_test.go +++ b/persistence/artist_repository_test.go @@ -1,6 +1,7 @@ package persistence import ( + "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/model" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -10,7 +11,7 @@ var _ = Describe("ArtistRepository", func() { var repo model.ArtistRepository BeforeEach(func() { - repo = NewArtistRepository() + repo = NewArtistRepository(orm.NewOrm()) }) Describe("Put/Get", func() { diff --git a/persistence/checksum_repository.go b/persistence/checksum_repository.go index a9be175dd..0c63cc37b 100644 --- a/persistence/checksum_repository.go +++ b/persistence/checksum_repository.go @@ -6,6 +6,7 @@ import ( ) type checkSumRepository struct { + ormer orm.Ormer } const checkSumId = "1" @@ -15,8 +16,8 @@ type checksum struct { Sum string } -func NewCheckSumRepository() model.ChecksumRepository { - r := &checkSumRepository{} +func NewCheckSumRepository(o orm.Ormer) model.ChecksumRepository { + r := &checkSumRepository{ormer: o} return r } @@ -24,7 +25,7 @@ func (r *checkSumRepository) GetData() (model.ChecksumMap, error) { loadedData := make(map[string]string) var all []checksum - _, err := Db().QueryTable(&checksum{}).Limit(-1).All(&all) + _, err := r.ormer.QueryTable(&checksum{}).Limit(-1).All(&all) if err != nil { return nil, err } @@ -37,24 +38,17 @@ func (r *checkSumRepository) GetData() (model.ChecksumMap, error) { } func (r *checkSumRepository) SetData(newSums model.ChecksumMap) error { - err := withTx(func(o orm.Ormer) error { - _, err := Db().Raw("delete from checksum").Exec() - if err != nil { - return err - } + _, err := r.ormer.Raw("delete from checksum").Exec() + if err != nil { + return err + } - var checksums []checksum - for k, v := range newSums { - cks := checksum{ID: k, Sum: v} - checksums = append(checksums, cks) - } - _, err = Db().InsertMulti(batchSize, &checksums) - if err != nil { - return err - } - - return nil - }) + var checksums []checksum + for k, v := range newSums { + cks := checksum{ID: k, Sum: v} + checksums = append(checksums, cks) + } + _, err = r.ormer.InsertMulti(batchSize, &checksums) if err != nil { return err } diff --git a/persistence/checksum_repository_test.go b/persistence/checksum_repository_test.go index 2bb32ea8c..babb3de5c 100644 --- a/persistence/checksum_repository_test.go +++ b/persistence/checksum_repository_test.go @@ -1,6 +1,7 @@ package persistence import ( + "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/model" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -10,8 +11,7 @@ var _ = Describe("ChecksumRepository", func() { var repo model.ChecksumRepository BeforeEach(func() { - Db().Delete(&checksum{ID: checkSumId}) - repo = NewCheckSumRepository() + repo = NewCheckSumRepository(orm.NewOrm()) err := repo.SetData(map[string]string{ "a": "AAA", "b": "BBB", }) @@ -27,7 +27,7 @@ var _ = Describe("ChecksumRepository", func() { }) It("persists data", func() { - newRepo := NewCheckSumRepository() + newRepo := NewCheckSumRepository(orm.NewOrm()) sums, err := newRepo.GetData() Expect(err).To(BeNil()) Expect(sums["b"]).To(Equal("BBB")) diff --git a/persistence/genre_repository.go b/persistence/genre_repository.go index 0d99ef594..edb17f2aa 100644 --- a/persistence/genre_repository.go +++ b/persistence/genre_repository.go @@ -7,19 +7,20 @@ import ( "github.com/cloudsonic/sonic-server/model" ) -type genreRepository struct{} +type genreRepository struct { + ormer orm.Ormer +} -func NewGenreRepository() model.GenreRepository { - return &genreRepository{} +func NewGenreRepository(o orm.Ormer) model.GenreRepository { + return &genreRepository{ormer: o} } func (r genreRepository) GetAll() (model.Genres, error) { - o := Db() genres := make(map[string]model.Genre) // Collect SongCount var res []orm.Params - _, err := o.Raw("select genre, count(*) as c from media_file group by genre").Values(&res) + _, err := r.ormer.Raw("select genre, count(*) as c from media_file group by genre").Values(&res) if err != nil { return nil, err } @@ -35,7 +36,7 @@ func (r genreRepository) GetAll() (model.Genres, error) { } // Collect AlbumCount - _, err = o.Raw("select genre, count(*) as c from album group by genre").Values(&res) + _, err = r.ormer.Raw("select genre, count(*) as c from album group by genre").Values(&res) if err != nil { return nil, err } diff --git a/persistence/genre_repository_test.go b/persistence/genre_repository_test.go index 24a9756a2..1e82f3dd1 100644 --- a/persistence/genre_repository_test.go +++ b/persistence/genre_repository_test.go @@ -1,6 +1,7 @@ package persistence import ( + "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/model" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -10,7 +11,7 @@ var _ = Describe("GenreRepository", func() { var repo model.GenreRepository BeforeEach(func() { - repo = NewGenreRepository() + repo = NewGenreRepository(orm.NewOrm()) }) It("returns all records", func() { diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index b8b1d93d5..2bde5af01 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -41,28 +41,27 @@ type mediaFileRepository struct { searchableRepository } -func NewMediaFileRepository() model.MediaFileRepository { +func NewMediaFileRepository(o orm.Ormer) model.MediaFileRepository { r := &mediaFileRepository{} + r.ormer = o r.tableName = "media_file" return r } func (r *mediaFileRepository) Put(m *model.MediaFile, overrideAnnotation bool) error { tm := mediaFile(*m) - return withTx(func(o orm.Ormer) error { - if !overrideAnnotation { - // Don't update media annotation fields (playcount, starred, etc..) - return r.put(o, m.ID, m.Title, &tm, "path", "title", "album", "artist", "artist_id", "album_artist", - "album_id", "has_cover_art", "track_number", "disc_number", "year", "size", "suffix", "duration", - "bit_rate", "genre", "compilation", "updated_at") - } - return r.put(o, m.ID, m.Title, &tm) - }) + if !overrideAnnotation { + // Don't update media annotation fields (playcount, starred, etc..) + return r.put(m.ID, m.Title, &tm, "path", "title", "album", "artist", "artist_id", "album_artist", + "album_id", "has_cover_art", "track_number", "disc_number", "year", "size", "suffix", "duration", + "bit_rate", "genre", "compilation", "updated_at") + } + return r.put(m.ID, m.Title, &tm) } func (r *mediaFileRepository) Get(id string) (*model.MediaFile, error) { tm := mediaFile{ID: id} - err := Db().Read(&tm) + err := r.ormer.Read(&tm) if err == orm.ErrNoRows { return nil, model.ErrNotFound } @@ -83,7 +82,7 @@ func (r *mediaFileRepository) toMediaFiles(all []mediaFile) model.MediaFiles { func (r *mediaFileRepository) FindByAlbum(albumId string) (model.MediaFiles, error) { var mfs []mediaFile - _, err := r.newQuery(Db()).Filter("album_id", albumId).OrderBy("disc_number", "track_number").All(&mfs) + _, err := r.newQuery().Filter("album_id", albumId).OrderBy("disc_number", "track_number").All(&mfs) if err != nil { return nil, err } @@ -92,7 +91,7 @@ func (r *mediaFileRepository) FindByAlbum(albumId string) (model.MediaFiles, err func (r *mediaFileRepository) FindByPath(path string) (model.MediaFiles, error) { var mfs []mediaFile - _, err := r.newQuery(Db()).Filter("path__istartswith", path).OrderBy("disc_number", "track_number").All(&mfs) + _, err := r.newQuery().Filter("path__istartswith", path).OrderBy("disc_number", "track_number").All(&mfs) if err != nil { return nil, err } @@ -109,10 +108,9 @@ func (r *mediaFileRepository) FindByPath(path string) (model.MediaFiles, error) } func (r *mediaFileRepository) DeleteByPath(path string) error { - o := Db() var mfs []mediaFile // TODO Paginate this (and all other situations similar) - _, err := r.newQuery(o).Filter("path__istartswith", path).OrderBy("disc_number", "track_number").All(&mfs) + _, err := r.newQuery().Filter("path__istartswith", path).OrderBy("disc_number", "track_number").All(&mfs) if err != nil { return err } @@ -128,13 +126,13 @@ func (r *mediaFileRepository) DeleteByPath(path string) error { if len(filtered) == 0 { return nil } - _, err = r.newQuery(o).Filter("id__in", filtered).Delete() + _, err = r.newQuery().Filter("id__in", filtered).Delete() return err } func (r *mediaFileRepository) GetStarred(options ...model.QueryOptions) (model.MediaFiles, error) { var starred []mediaFile - _, err := r.newQuery(Db(), options...).Filter("starred", true).All(&starred) + _, err := r.newQuery(options...).Filter("starred", true).All(&starred) if err != nil { return nil, err } @@ -149,7 +147,7 @@ func (r *mediaFileRepository) SetStar(starred bool, ids ...string) error { if starred { starredAt = time.Now() } - _, err := r.newQuery(Db()).Filter("id__in", ids).Update(orm.Params{ + _, err := r.newQuery().Filter("id__in", ids).Update(orm.Params{ "starred": starred, "starred_at": starredAt, }) @@ -160,12 +158,12 @@ func (r *mediaFileRepository) SetRating(rating int, ids ...string) error { if len(ids) == 0 { return model.ErrNotFound } - _, err := r.newQuery(Db()).Filter("id__in", ids).Update(orm.Params{"rating": rating}) + _, err := r.newQuery().Filter("id__in", ids).Update(orm.Params{"rating": rating}) return err } func (r *mediaFileRepository) MarkAsPlayed(id string, playDate time.Time) error { - _, err := r.newQuery(Db()).Filter("id", id).Update(orm.Params{ + _, err := r.newQuery().Filter("id", id).Update(orm.Params{ "play_count": orm.ColValue(orm.ColAdd, 1), "play_date": playDate, }) @@ -173,12 +171,10 @@ func (r *mediaFileRepository) MarkAsPlayed(id string, playDate time.Time) error } func (r *mediaFileRepository) PurgeInactive(activeList model.MediaFiles) error { - return withTx(func(o orm.Ormer) error { - _, err := r.purgeInactive(o, activeList, func(item interface{}) string { - return item.(model.MediaFile).ID - }) - return err + _, err := r.purgeInactive(activeList, func(item interface{}) string { + return item.(model.MediaFile).ID }) + return err } func (r *mediaFileRepository) Search(q string, offset int, size int) (model.MediaFiles, error) { diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go index 4a42feca7..3326d878b 100644 --- a/persistence/mediafile_repository_test.go +++ b/persistence/mediafile_repository_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" + "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/model" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -13,7 +14,7 @@ var _ = Describe("MediaFileRepository", func() { var repo model.MediaFileRepository BeforeEach(func() { - repo = NewMediaFileRepository() + repo = NewMediaFileRepository(orm.NewOrm()) }) Describe("FindByPath", func() { diff --git a/persistence/mediafolders_repository.go b/persistence/mediafolders_repository.go index 5ddc4388a..364e15c7a 100644 --- a/persistence/mediafolders_repository.go +++ b/persistence/mediafolders_repository.go @@ -1,6 +1,7 @@ package persistence import ( + "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/conf" "github.com/cloudsonic/sonic-server/model" ) @@ -9,17 +10,13 @@ type mediaFolderRepository struct { model.MediaFolderRepository } -func NewMediaFolderRepository() model.MediaFolderRepository { +func NewMediaFolderRepository(o orm.Ormer) model.MediaFolderRepository { return &mediaFolderRepository{} } func (*mediaFolderRepository) GetAll() (model.MediaFolders, error) { mediaFolder := model.MediaFolder{ID: "0", Path: conf.Sonic.MusicFolder} - if conf.Sonic.DevUseFileScanner { - mediaFolder.Name = "Music Library" - } else { - mediaFolder.Name = "iTunes Library" - } + mediaFolder.Name = "Music Library" result := make(model.MediaFolders, 1) result[0] = mediaFolder return result, nil diff --git a/persistence/mock_persistence.go b/persistence/mock_persistence.go new file mode 100644 index 000000000..71738165e --- /dev/null +++ b/persistence/mock_persistence.go @@ -0,0 +1,54 @@ +package persistence + +import "github.com/cloudsonic/sonic-server/model" + +type MockDataStore struct { + MockedGenre model.GenreRepository + MockedAlbum model.AlbumRepository + MockedArtist model.ArtistRepository + MockedMediaFile model.MediaFileRepository +} + +func (db *MockDataStore) Album() model.AlbumRepository { + if db.MockedAlbum == nil { + db.MockedAlbum = CreateMockAlbumRepo() + } + return db.MockedAlbum +} + +func (db *MockDataStore) Artist() model.ArtistRepository { + if db.MockedArtist == nil { + db.MockedArtist = CreateMockArtistRepo() + } + return db.MockedArtist +} + +func (db *MockDataStore) MediaFile() model.MediaFileRepository { + if db.MockedMediaFile == nil { + db.MockedMediaFile = CreateMockMediaFileRepo() + } + return db.MockedMediaFile +} + +func (db *MockDataStore) MediaFolder() model.MediaFolderRepository { + return struct{ model.MediaFolderRepository }{} +} + +func (db *MockDataStore) Genre() model.GenreRepository { + if db.MockedGenre != nil { + return db.MockedGenre + } + return struct{ model.GenreRepository }{} +} + +func (db *MockDataStore) Playlist() model.PlaylistRepository { + return struct{ model.PlaylistRepository }{} +} + +func (db *MockDataStore) Property() model.PropertyRepository { + return struct{ model.PropertyRepository }{} +} + +func (db *MockDataStore) WithTx(block func(db model.DataStore) error) error { + return block(db) +} diff --git a/persistence/persistence.go b/persistence/persistence.go index d269476a4..62c502c81 100644 --- a/persistence/persistence.go +++ b/persistence/persistence.go @@ -8,6 +8,7 @@ import ( "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/conf" "github.com/cloudsonic/sonic-server/log" + "github.com/cloudsonic/sonic-server/model" _ "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" ) @@ -19,7 +20,11 @@ var ( driver = "sqlite3" ) -func Db() orm.Ormer { +type SQLStore struct { + orm orm.Ormer +} + +func New() model.DataStore { once.Do(func() { dbPath := conf.Sonic.DbPath if dbPath == ":memory:" { @@ -31,17 +36,47 @@ func Db() orm.Ormer { } log.Debug("Opening DB from: "+dbPath, "driver", driver) }) - return orm.NewOrm() + return &SQLStore{} } -func withTx(block func(orm.Ormer) error) error { +func (db *SQLStore) Album() model.AlbumRepository { + return NewAlbumRepository(db.getOrmer()) +} + +func (db *SQLStore) Artist() model.ArtistRepository { + return NewArtistRepository(db.getOrmer()) +} + +func (db *SQLStore) MediaFile() model.MediaFileRepository { + return NewMediaFileRepository(db.getOrmer()) +} + +func (db *SQLStore) MediaFolder() model.MediaFolderRepository { + return NewMediaFolderRepository(db.getOrmer()) +} + +func (db *SQLStore) Genre() model.GenreRepository { + return NewGenreRepository(db.getOrmer()) +} + +func (db *SQLStore) Playlist() model.PlaylistRepository { + return NewPlaylistRepository(db.getOrmer()) +} + +func (db *SQLStore) Property() model.PropertyRepository { + return NewPropertyRepository(db.getOrmer()) +} + +func (db *SQLStore) WithTx(block func(tx model.DataStore) error) error { o := orm.NewOrm() err := o.Begin() if err != nil { return err } - err = block(o) + newDb := &SQLStore{orm: o} + err = block(newDb) + if err != nil { err2 := o.Rollback() if err2 != nil { @@ -57,15 +92,11 @@ func withTx(block func(orm.Ormer) error) error { return nil } -func collectField(collection interface{}, getValue func(item interface{}) string) []string { - s := reflect.ValueOf(collection) - result := make([]string, s.Len()) - - for i := 0; i < s.Len(); i++ { - result[i] = getValue(s.Index(i).Interface()) +func (db *SQLStore) getOrmer() orm.Ormer { + if db.orm == nil { + return orm.NewOrm() } - - return result + return db.orm } func initORM(dbPath string) error { @@ -87,3 +118,14 @@ func initORM(dbPath string) error { } return orm.RunSyncdb("default", false, verbose) } + +func collectField(collection interface{}, getValue func(item interface{}) string) []string { + s := reflect.ValueOf(collection) + result := make([]string, s.Len()) + + for i := 0; i < s.Len(); i++ { + result[i] = getValue(s.Index(i).Interface()) + } + + return result +} diff --git a/persistence/persistence_suite_test.go b/persistence/persistence_suite_test.go index da3d40e8a..70edb1fb2 100644 --- a/persistence/persistence_suite_test.go +++ b/persistence/persistence_suite_test.go @@ -57,19 +57,19 @@ var _ = Describe("Initialize test DB", func() { //conf.Sonic.DbPath, _ = ioutil.TempDir("", "cloudsonic_tests") //os.MkdirAll(conf.Sonic.DbPath, 0700) conf.Sonic.DbPath = ":memory:" - Db() - artistRepo := NewArtistRepository() + ds := New() + artistRepo := ds.Artist() for _, a := range testArtists { artistRepo.Put(&a) } - albumRepository := NewAlbumRepository() + albumRepository := ds.Album() for _, a := range testAlbums { err := albumRepository.Put(&a) if err != nil { panic(err) } } - mediaFileRepository := NewMediaFileRepository() + mediaFileRepository := ds.MediaFile() for _, s := range testSongs { err := mediaFileRepository.Put(&s, true) if err != nil { diff --git a/persistence/playlist_repository.go b/persistence/playlist_repository.go index 59b74b8b2..39ec75d74 100644 --- a/persistence/playlist_repository.go +++ b/persistence/playlist_repository.go @@ -22,22 +22,21 @@ type playlistRepository struct { sqlRepository } -func NewPlaylistRepository() model.PlaylistRepository { +func NewPlaylistRepository(o orm.Ormer) model.PlaylistRepository { r := &playlistRepository{} + r.ormer = o r.tableName = "playlist" return r } func (r *playlistRepository) Put(p *model.Playlist) error { tp := r.fromDomain(p) - return withTx(func(o orm.Ormer) error { - return r.put(o, p.ID, &tp) - }) + return r.put(p.ID, &tp) } func (r *playlistRepository) Get(id string) (*model.Playlist, error) { tp := &playlist{ID: id} - err := Db().Read(tp) + err := r.ormer.Read(tp) if err == orm.ErrNoRows { return nil, model.ErrNotFound } @@ -50,7 +49,7 @@ func (r *playlistRepository) Get(id string) (*model.Playlist, error) { func (r *playlistRepository) GetAll(options ...model.QueryOptions) (model.Playlists, error) { var all []playlist - _, err := r.newQuery(Db(), options...).All(&all) + _, err := r.newQuery(options...).All(&all) if err != nil { return nil, err } @@ -66,12 +65,10 @@ func (r *playlistRepository) toPlaylists(all []playlist) (model.Playlists, error } func (r *playlistRepository) PurgeInactive(activeList model.Playlists) ([]string, error) { - return nil, withTx(func(o orm.Ormer) error { - _, err := r.purgeInactive(o, activeList, func(item interface{}) string { - return item.(model.Playlist).ID - }) - return err + _, err := r.purgeInactive(activeList, func(item interface{}) string { + return item.(model.Playlist).ID }) + return nil, err } func (r *playlistRepository) toDomain(p *playlist) model.Playlist { diff --git a/persistence/property_repository.go b/persistence/property_repository.go index 96046f61a..091293474 100644 --- a/persistence/property_repository.go +++ b/persistence/property_repository.go @@ -14,27 +14,28 @@ type propertyRepository struct { sqlRepository } -func NewPropertyRepository() model.PropertyRepository { +func NewPropertyRepository(o orm.Ormer) model.PropertyRepository { r := &propertyRepository{} + r.ormer = o r.tableName = "property" return r } func (r *propertyRepository) Put(id string, value string) error { p := &property{ID: id, Value: value} - num, err := Db().Update(p) + num, err := r.ormer.Update(p) if err != nil { return nil } if num == 0 { - _, err = Db().Insert(p) + _, err = r.ormer.Insert(p) } return err } func (r *propertyRepository) Get(id string) (string, error) { p := &property{ID: id} - err := Db().Read(p) + err := r.ormer.Read(p) if err == orm.ErrNoRows { return "", model.ErrNotFound } diff --git a/persistence/property_repository_test.go b/persistence/property_repository_test.go index 095fe1145..bc95d9ac2 100644 --- a/persistence/property_repository_test.go +++ b/persistence/property_repository_test.go @@ -1,6 +1,7 @@ package persistence import ( + "github.com/astaxie/beego/orm" "github.com/cloudsonic/sonic-server/model" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -10,7 +11,7 @@ var _ = Describe("PropertyRepository", func() { var repo model.PropertyRepository BeforeEach(func() { - repo = NewPropertyRepository() + repo = NewPropertyRepository(orm.NewOrm()) repo.(*propertyRepository).DeleteAll() }) diff --git a/persistence/searchable_repository.go b/persistence/searchable_repository.go index 3f7f458d1..fb12cbc95 100644 --- a/persistence/searchable_repository.go +++ b/persistence/searchable_repository.go @@ -20,59 +20,57 @@ type searchableRepository struct { } func (r *searchableRepository) DeleteAll() error { - return withTx(func(o orm.Ormer) error { - _, err := r.newQuery(Db()).Filter("id__isnull", false).Delete() - if err != nil { - return err - } - return r.removeAllFromIndex(o, r.tableName) - }) + _, err := r.newQuery().Filter("id__isnull", false).Delete() + if err != nil { + return err + } + return r.removeAllFromIndex(r.ormer, r.tableName) } -func (r *searchableRepository) put(o orm.Ormer, id string, textToIndex string, a interface{}, fields ...string) error { - c, err := r.newQuery(o).Filter("id", id).Count() +func (r *searchableRepository) put(id string, textToIndex string, a interface{}, fields ...string) error { + c, err := r.newQuery().Filter("id", id).Count() if err != nil { return err } if c == 0 { - err = r.insert(o, a) + err = r.insert(a) if err != nil && err.Error() == "LastInsertId is not supported by this driver" { err = nil } } else { - _, err = o.Update(a, fields...) + _, err = r.ormer.Update(a, fields...) } if err != nil { return err } - return r.addToIndex(o, r.tableName, id, textToIndex) + return r.addToIndex(r.tableName, id, textToIndex) } -func (r *searchableRepository) purgeInactive(o orm.Ormer, activeList interface{}, getId func(item interface{}) string) ([]string, error) { - idsToDelete, err := r.sqlRepository.purgeInactive(o, activeList, getId) +func (r *searchableRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) { + idsToDelete, err := r.sqlRepository.purgeInactive(activeList, getId) if err != nil { return nil, err } - return idsToDelete, r.removeFromIndex(o, r.tableName, idsToDelete) + return idsToDelete, r.removeFromIndex(r.tableName, idsToDelete) } -func (r *searchableRepository) addToIndex(o orm.Ormer, table, id, text string) error { +func (r *searchableRepository) addToIndex(table, id, text string) error { item := Search{ID: id, Table: table} - err := o.Read(&item) + err := r.ormer.Read(&item) if err != nil && err != orm.ErrNoRows { return err } sanitizedText := strings.TrimSpace(sanitize.Accents(strings.ToLower(text))) item = Search{ID: id, Table: table, FullText: sanitizedText} if err == orm.ErrNoRows { - err = r.insert(o, &item) + err = r.insert(&item) } else { - _, err = o.Update(&item) + _, err = r.ormer.Update(&item) } return err } -func (r *searchableRepository) removeFromIndex(o orm.Ormer, table string, ids []string) error { +func (r *searchableRepository) removeFromIndex(table string, ids []string) error { var offset int for { var subset = paginateSlice(ids, offset, batchSize) @@ -81,7 +79,7 @@ func (r *searchableRepository) removeFromIndex(o orm.Ormer, table string, ids [] } log.Trace("Deleting searchable items", "table", table, "num", len(subset), "from", offset) offset += len(subset) - _, err := o.QueryTable(&Search{}).Filter("table", table).Filter("id__in", subset).Delete() + _, err := r.ormer.QueryTable(&Search{}).Filter("table", table).Filter("id__in", subset).Delete() if err != nil { return err } @@ -116,6 +114,6 @@ func (r *searchableRepository) doSearch(table string, q string, offset, size int if err != nil { return err } - _, err = Db().Raw(sql, args...).QueryRows(results) + _, err = r.ormer.Raw(sql, args...).QueryRows(results) return err } diff --git a/persistence/sql_repository.go b/persistence/sql_repository.go index 4a184c880..6e2008164 100644 --- a/persistence/sql_repository.go +++ b/persistence/sql_repository.go @@ -8,10 +8,11 @@ import ( type sqlRepository struct { tableName string + ormer orm.Ormer } -func (r *sqlRepository) newQuery(o orm.Ormer, options ...model.QueryOptions) orm.QuerySeter { - q := o.QueryTable(r.tableName) +func (r *sqlRepository) newQuery(options ...model.QueryOptions) orm.QuerySeter { + q := r.ormer.QueryTable(r.tableName) if len(options) > 0 { opts := options[0] q = q.Offset(opts.Offset) @@ -30,17 +31,17 @@ func (r *sqlRepository) newQuery(o orm.Ormer, options ...model.QueryOptions) orm } func (r *sqlRepository) CountAll() (int64, error) { - return r.newQuery(Db()).Count() + return r.newQuery().Count() } func (r *sqlRepository) Exists(id string) (bool, error) { - c, err := r.newQuery(Db()).Filter("id", id).Count() + c, err := r.newQuery().Filter("id", id).Count() return c == 1, err } // TODO This is used to generate random lists. Can be optimized in SQL: https://stackoverflow.com/a/19419 func (r *sqlRepository) GetAllIds() ([]string, error) { - qs := r.newQuery(Db()) + qs := r.newQuery() var values []orm.Params num, err := qs.Values(&values, "id") if num == 0 { @@ -55,27 +56,27 @@ func (r *sqlRepository) GetAllIds() ([]string, error) { } // "Hack" to bypass Postgres driver limitation -func (r *sqlRepository) insert(o orm.Ormer, record interface{}) error { - _, err := o.Insert(record) +func (r *sqlRepository) insert(record interface{}) error { + _, err := r.ormer.Insert(record) if err != nil && err.Error() != "LastInsertId is not supported by this driver" { return err } return nil } -func (r *sqlRepository) put(o orm.Ormer, id string, a interface{}) error { - c, err := r.newQuery(o).Filter("id", id).Count() +func (r *sqlRepository) put(id string, a interface{}) error { + c, err := r.newQuery().Filter("id", id).Count() if err != nil { return err } if c == 0 { - err = r.insert(o, a) + err = r.insert(a) if err != nil && err.Error() == "LastInsertId is not supported by this driver" { err = nil } return err } - _, err = o.Update(a) + _, err = r.ormer.Update(a) return err } @@ -113,18 +114,16 @@ func difference(slice1 []string, slice2 []string) []string { } func (r *sqlRepository) Delete(id string) error { - _, err := r.newQuery(Db()).Filter("id", id).Delete() + _, err := r.newQuery().Filter("id", id).Delete() return err } func (r *sqlRepository) DeleteAll() error { - return withTx(func(o orm.Ormer) error { - _, err := r.newQuery(Db()).Filter("id__isnull", false).Delete() - return err - }) + _, err := r.newQuery().Filter("id__isnull", false).Delete() + return err } -func (r *sqlRepository) purgeInactive(o orm.Ormer, activeList interface{}, getId func(item interface{}) string) ([]string, error) { +func (r *sqlRepository) purgeInactive(activeList interface{}, getId func(item interface{}) string) ([]string, error) { allIds, err := r.GetAllIds() if err != nil { return nil, err @@ -144,7 +143,7 @@ func (r *sqlRepository) purgeInactive(o orm.Ormer, activeList interface{}, getId } log.Trace("-- Purging inactive records", "table", r.tableName, "num", len(subset), "from", offset) offset += len(subset) - _, err := r.newQuery(o).Filter("id__in", subset).Delete() + _, err := r.newQuery().Filter("id__in", subset).Delete() if err != nil { return nil, err } diff --git a/persistence/wire_provider.go b/persistence/wire_provider.go index e680e7e7e..d2cc16f13 100644 --- a/persistence/wire_provider.go +++ b/persistence/wire_provider.go @@ -5,12 +5,13 @@ import ( ) var Set = wire.NewSet( - NewArtistRepository, - NewMediaFileRepository, - NewAlbumRepository, - NewCheckSumRepository, - NewPropertyRepository, - NewPlaylistRepository, - NewMediaFolderRepository, - NewGenreRepository, + //NewArtistRepository, + //NewMediaFileRepository, + //NewAlbumRepository, + //NewCheckSumRepository, + //NewPropertyRepository, + //NewPlaylistRepository, + //NewMediaFolderRepository, + //NewGenreRepository, + New, ) diff --git a/scanner/scanner.go b/scanner/scanner.go index cc99a53d4..2319d5c73 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -13,28 +13,11 @@ import ( type Scanner struct { folders map[string]FolderScanner - repos Repositories + ds model.DataStore } -type Repositories struct { - folder model.MediaFolderRepository - mediaFile model.MediaFileRepository - album model.AlbumRepository - artist model.ArtistRepository - playlist model.PlaylistRepository - property model.PropertyRepository -} - -func New(mfRepo model.MediaFileRepository, albumRepo model.AlbumRepository, artistRepo model.ArtistRepository, plsRepo model.PlaylistRepository, folderRepo model.MediaFolderRepository, property model.PropertyRepository) *Scanner { - repos := Repositories{ - folder: folderRepo, - mediaFile: mfRepo, - album: albumRepo, - artist: artistRepo, - playlist: plsRepo, - property: property, - } - s := &Scanner{repos: repos, folders: map[string]FolderScanner{}} +func New(ds model.DataStore) *Scanner { + s := &Scanner{ds: ds, folders: map[string]FolderScanner{}} s.loadFolders() return s } @@ -77,7 +60,7 @@ func (s *Scanner) RescanAll(fullRescan bool) error { func (s *Scanner) Status() []StatusInfo { return nil } func (s *Scanner) getLastModifiedSince(folder string) time.Time { - ms, err := s.repos.property.Get(model.PropLastScan + "-" + folder) + ms, err := s.ds.Property().Get(model.PropLastScan + "-" + folder) if err != nil { return time.Time{} } @@ -90,14 +73,14 @@ func (s *Scanner) getLastModifiedSince(folder string) time.Time { func (s *Scanner) updateLastModifiedSince(folder string, t time.Time) { millis := t.UnixNano() / int64(time.Millisecond) - s.repos.property.Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis)) + s.ds.Property().Put(model.PropLastScan+"-"+folder, fmt.Sprint(millis)) } func (s *Scanner) loadFolders() { - fs, _ := s.repos.folder.GetAll() + fs, _ := s.ds.MediaFolder().GetAll() for _, f := range fs { log.Info("Configuring Media Folder", "name", f.Name, "path", f.Path) - s.folders[f.Path] = NewTagScanner(f.Path, s.repos) + s.folders[f.Path] = NewTagScanner(f.Path, s.ds) } } diff --git a/scanner/scanner_suite_test.go b/scanner/scanner_suite_test.go index 314ead750..2adf1a356 100644 --- a/scanner/scanner_suite_test.go +++ b/scanner/scanner_suite_test.go @@ -21,16 +21,10 @@ func xTestScanner(t *testing.T) { var _ = Describe("TODO: REMOVE", func() { conf.Sonic.DbPath = "./testDB" log.SetLevel(log.LevelDebug) - repos := Repositories{ - folder: persistence.NewMediaFolderRepository(), - mediaFile: persistence.NewMediaFileRepository(), - album: persistence.NewAlbumRepository(), - artist: persistence.NewArtistRepository(), - playlist: nil, - } + ds := persistence.New() It("WORKS!", func() { - t := NewTagScanner("/Users/deluan/Music/iTunes/iTunes Media/Music", repos) - //t := NewTagScanner("/Users/deluan/Development/cloudsonic/sonic-server/tests/fixtures", repos) + t := NewTagScanner("/Users/deluan/Music/iTunes/iTunes Media/Music", ds) + //t := NewTagScanner("/Users/deluan/Development/cloudsonic/sonic-server/tests/fixtures", ds) Expect(t.Scan(nil, time.Time{})).To(BeNil()) }) }) diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.go index 4a3f60d2b..8aba6d702 100644 --- a/scanner/tag_scanner.go +++ b/scanner/tag_scanner.go @@ -18,14 +18,14 @@ import ( type TagScanner struct { rootFolder string - repos Repositories + ds model.DataStore detector *ChangeDetector } -func NewTagScanner(rootFolder string, repos Repositories) *TagScanner { +func NewTagScanner(rootFolder string, ds model.DataStore) *TagScanner { return &TagScanner{ rootFolder: rootFolder, - repos: repos, + ds: ds, detector: NewChangeDetector(rootFolder), } } @@ -105,12 +105,12 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time) erro return err } - err = s.repos.album.PurgeEmpty() + err = s.ds.Album().PurgeEmpty() if err != nil { return err } - err = s.repos.artist.PurgeEmpty() + err = s.ds.Artist().PurgeEmpty() if err != nil { return err } @@ -123,7 +123,7 @@ func (s *TagScanner) refreshAlbums(updatedAlbums map[string]bool) error { for id := range updatedAlbums { ids = append(ids, id) } - return s.repos.album.Refresh(ids...) + return s.ds.Album().Refresh(ids...) } func (s *TagScanner) refreshArtists(updatedArtists map[string]bool) error { @@ -131,7 +131,7 @@ func (s *TagScanner) refreshArtists(updatedArtists map[string]bool) error { for id := range updatedArtists { ids = append(ids, id) } - return s.repos.artist.Refresh(ids...) + return s.ds.Artist().Refresh(ids...) } func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error { @@ -141,7 +141,7 @@ func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]boo // Load folder's current tracks from DB into a map currentTracks := map[string]model.MediaFile{} - ct, err := s.repos.mediaFile.FindByPath(dir) + ct, err := s.ds.MediaFile().FindByPath(dir) if err != nil { return err } @@ -169,7 +169,7 @@ func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]boo for _, n := range newTracks { c, ok := currentTracks[n.ID] if !ok || (ok && n.UpdatedAt.After(c.UpdatedAt)) { - err := s.repos.mediaFile.Put(&n, false) + err := s.ds.MediaFile().Put(&n, false) updatedArtists[n.ArtistID] = true updatedAlbums[n.AlbumID] = true numUpdatedTracks++ @@ -183,7 +183,7 @@ func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]boo // Remaining tracks from DB that are not in the folder are deleted for id := range currentTracks { numPurgedTracks++ - if err := s.repos.mediaFile.Delete(id); err != nil { + if err := s.ds.MediaFile().Delete(id); err != nil { return err } } @@ -195,7 +195,7 @@ func (s *TagScanner) processChangedDir(dir string, updatedArtists map[string]boo func (s *TagScanner) processDeletedDir(dir string, updatedArtists map[string]bool, updatedAlbums map[string]bool) error { dir = path.Join(s.rootFolder, dir) - ct, err := s.repos.mediaFile.FindByPath(dir) + ct, err := s.ds.MediaFile().FindByPath(dir) if err != nil { return err } @@ -204,7 +204,7 @@ func (s *TagScanner) processDeletedDir(dir string, updatedArtists map[string]boo updatedAlbums[t.AlbumID] = true } - return s.repos.mediaFile.DeleteByPath(dir) + return s.ds.MediaFile().DeleteByPath(dir) } func (s *TagScanner) loadTracks(dirPath string) (model.MediaFiles, error) { diff --git a/scanner_legacy/importer.go b/scanner_legacy/importer.go deleted file mode 100644 index 749023dad..000000000 --- a/scanner_legacy/importer.go +++ /dev/null @@ -1,249 +0,0 @@ -package scanner_legacy - -import ( - "fmt" - "os" - "strconv" - "time" - - "github.com/cloudsonic/sonic-server/conf" - "github.com/cloudsonic/sonic-server/log" - "github.com/cloudsonic/sonic-server/model" -) - -type Scanner interface { - ScanLibrary(lastModifiedSince time.Time, path string) (int, error) - MediaFiles() map[string]*model.MediaFile - Albums() map[string]*model.Album - Artists() map[string]*model.Artist - Playlists() map[string]*model.Playlist -} - -type Importer struct { - scanner Scanner - mediaFolder string - mfRepo model.MediaFileRepository - albumRepo model.AlbumRepository - artistRepo model.ArtistRepository - plsRepo model.PlaylistRepository - propertyRepo model.PropertyRepository - lastScan time.Time - lastCheck time.Time -} - -func NewImporter(mediaFolder string, scanner Scanner, mfRepo model.MediaFileRepository, albumRepo model.AlbumRepository, artistRepo model.ArtistRepository, plsRepo model.PlaylistRepository, propertyRepo model.PropertyRepository) *Importer { - return &Importer{ - scanner: scanner, - mediaFolder: mediaFolder, - mfRepo: mfRepo, - albumRepo: albumRepo, - artistRepo: artistRepo, - plsRepo: plsRepo, - propertyRepo: propertyRepo, - } -} - -func (i *Importer) CheckForUpdates(force bool) { - if force { - i.lastCheck = time.Time{} - } - - i.startImport() -} - -func (i *Importer) startImport() { - go func() { - info, err := os.Stat(i.mediaFolder) - if err != nil { - log.Error(err) - return - } - if i.lastCheck.After(info.ModTime()) { - return - } - i.lastCheck = time.Now() - - i.scan() - }() -} - -func (i *Importer) scan() { - i.lastScan = i.lastModifiedSince() - - if i.lastScan.IsZero() { - log.Info("Starting first iTunes Library scan. This can take a while...") - } - - total, err := i.scanner.ScanLibrary(i.lastScan, i.mediaFolder) - if err != nil { - log.Error("Error importing iTunes Library", err) - return - } - - log.Debug("Totals informed by the scanner", "tracks", total, - "songs", len(i.scanner.MediaFiles()), - "albums", len(i.scanner.Albums()), - "artists", len(i.scanner.Artists()), - "playlists", len(i.scanner.Playlists())) - - if err := i.importLibrary(); err != nil { - log.Error("Error persisting data", err) - } - if i.lastScan.IsZero() { - log.Info("Finished first iTunes Library import") - } else { - log.Debug("Finished updating tracks from iTunes Library") - } -} - -func (i *Importer) lastModifiedSince() time.Time { - ms, err := i.propertyRepo.Get(model.PropLastScan) - if err != nil { - log.Warn("Couldn't read LastScan", err) - return time.Time{} - } - if ms == "" { - log.Debug("First scan") - return time.Time{} - } - s, _ := strconv.ParseInt(ms, 10, 64) - return time.Unix(0, s*int64(time.Millisecond)) -} - -func (i *Importer) importLibrary() (err error) { - arc, _ := i.artistRepo.CountAll() - alc, _ := i.albumRepo.CountAll() - mfc, _ := i.mfRepo.CountAll() - plc, _ := i.plsRepo.CountAll() - - log.Debug("Saving updated data") - mfs, mfu := i.importMediaFiles() - log.Debug("Imported media files", "total", len(mfs), "updated", mfu) - als, alu := i.importAlbums() - log.Debug("Imported albums", "total", len(als), "updated", alu) - ars := i.importArtists() - log.Debug("Imported artists", "total", len(ars)) - pls := i.importPlaylists() - log.Debug("Imported playlists", "total", len(pls)) - - log.Debug("Purging old data") - if err := i.mfRepo.PurgeInactive(mfs); err != nil { - log.Error(err) - } - if err := i.albumRepo.PurgeInactive(als); err != nil { - log.Error(err) - } - if err := i.artistRepo.PurgeInactive(ars); err != nil { - log.Error("Deleting inactive artists", err) - } - if _, err := i.plsRepo.PurgeInactive(pls); err != nil { - log.Error(err) - } - - arc2, _ := i.artistRepo.CountAll() - alc2, _ := i.albumRepo.CountAll() - mfc2, _ := i.mfRepo.CountAll() - plc2, _ := i.plsRepo.CountAll() - - if arc != arc2 || alc != alc2 || mfc != mfc2 || plc != plc2 { - log.Info(fmt.Sprintf("Updated library totals: %d(%+d) artists, %d(%+d) albums, %d(%+d) songs, %d(%+d) playlists", arc2, arc2-arc, alc2, alc2-alc, mfc2, mfc2-mfc, plc2, plc2-plc)) - } - if alu > 0 || mfu > 0 { - log.Info(fmt.Sprintf("Updated items: %d album(s), %d song(s)", alu, mfu)) - } - - if err == nil { - millis := time.Now().UnixNano() / int64(time.Millisecond) - i.propertyRepo.Put(model.PropLastScan, fmt.Sprint(millis)) - log.Debug("LastScan", "timestamp", millis) - } - - return err -} - -func (i *Importer) importMediaFiles() (model.MediaFiles, int) { - mfs := make(model.MediaFiles, len(i.scanner.MediaFiles())) - updates := 0 - j := 0 - for _, mf := range i.scanner.MediaFiles() { - mfs[j] = *mf - j++ - if mf.UpdatedAt.Before(i.lastScan) { - continue - } - if mf.Starred { - original, err := i.mfRepo.Get(mf.ID) - if err != nil || !original.Starred { - mf.StarredAt = mf.UpdatedAt - } else { - mf.StarredAt = original.StarredAt - } - } - if err := i.mfRepo.Put(mf, true); err != nil { - log.Error(err) - } - updates++ - if !i.lastScan.IsZero() { - log.Debug(fmt.Sprintf(`-- Updated Track: "%s"`, mf.Title)) - } - } - return mfs, updates -} - -func (i *Importer) importAlbums() (model.Albums, int) { - als := make(model.Albums, len(i.scanner.Albums())) - updates := 0 - j := 0 - for _, al := range i.scanner.Albums() { - als[j] = *al - j++ - if al.UpdatedAt.Before(i.lastScan) { - continue - } - if al.Starred { - original, err := i.albumRepo.Get(al.ID) - if err != nil || !original.Starred { - al.StarredAt = al.UpdatedAt - } else { - al.StarredAt = original.StarredAt - } - } - if err := i.albumRepo.Put(al); err != nil { - log.Error(err) - } - updates++ - if !i.lastScan.IsZero() { - log.Debug(fmt.Sprintf(`-- Updated Album: "%s" from "%s"`, al.Name, al.Artist)) - } - } - return als, updates -} - -func (i *Importer) importArtists() model.Artists { - ars := make(model.Artists, len(i.scanner.Artists())) - j := 0 - for _, ar := range i.scanner.Artists() { - ars[j] = *ar - j++ - if err := i.artistRepo.Put(ar); err != nil { - log.Error(err) - } - } - return ars -} - -func (i *Importer) importPlaylists() model.Playlists { - pls := make(model.Playlists, len(i.scanner.Playlists())) - j := 0 - for _, pl := range i.scanner.Playlists() { - pl.Public = true - pl.Owner = conf.Sonic.User - pl.Comment = "Original: " + pl.FullPath - pls[j] = *pl - j++ - if err := i.plsRepo.Put(pl); err != nil { - log.Error(err) - } - } - return pls -} diff --git a/scanner_legacy/itunes_scanner.go b/scanner_legacy/itunes_scanner.go deleted file mode 100644 index f8d1c3642..000000000 --- a/scanner_legacy/itunes_scanner.go +++ /dev/null @@ -1,407 +0,0 @@ -package scanner_legacy - -import ( - "crypto/md5" - "fmt" - "html" - "mime" - "net/url" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "time" - - "github.com/cloudsonic/sonic-server/conf" - "github.com/cloudsonic/sonic-server/log" - "github.com/cloudsonic/sonic-server/model" - "github.com/dhowden/itl" - "github.com/dhowden/tag" -) - -type ItunesScanner struct { - mediaFiles map[string]*model.MediaFile - albums map[string]*model.Album - artists map[string]*model.Artist - playlists map[string]*model.Playlist - pplaylists map[string]plsRelation - pmediaFiles map[int]*model.MediaFile - lastModifiedSince time.Time - checksumRepo model.ChecksumRepository - checksums model.ChecksumMap - newSums map[string]string -} - -func NewItunesScanner(checksumRepo model.ChecksumRepository) *ItunesScanner { - return &ItunesScanner{checksumRepo: checksumRepo} -} - -type plsRelation struct { - pID string - parentPID string - name string -} - -func (s *ItunesScanner) ScanLibrary(lastModifiedSince time.Time, path string) (int, error) { - log.Debug("Checking for updates", "lastModifiedSince", lastModifiedSince, "library", path) - xml, _ := os.Open(path) - l, err := itl.ReadFromXML(xml) - if err != nil { - return 0, err - } - log.Debug("Loaded tracks", "total", len(l.Tracks)) - - s.checksums, err = s.checksumRepo.GetData() - if err != nil { - log.Error("Error loading checksums", err) - s.checksums = map[string]string{} - } else { - log.Debug("Loaded checksums", "total", len(s.checksums)) - } - - s.lastModifiedSince = lastModifiedSince - s.mediaFiles = make(map[string]*model.MediaFile) - s.albums = make(map[string]*model.Album) - s.artists = make(map[string]*model.Artist) - s.playlists = make(map[string]*model.Playlist) - s.pplaylists = make(map[string]plsRelation) - s.pmediaFiles = make(map[int]*model.MediaFile) - s.newSums = make(map[string]string) - songsPerAlbum := make(map[string]int) - albumsPerArtist := make(map[string]map[string]bool) - - i := 0 - for _, t := range l.Tracks { - if !s.skipTrack(&t) { - s.calcCheckSum(&t) - - ar := s.collectArtists(&t) - mf := s.collectMediaFiles(&t) - s.collectAlbums(&t, mf, ar) - - songsPerAlbum[mf.AlbumID]++ - if albumsPerArtist[mf.ArtistID] == nil { - albumsPerArtist[mf.ArtistID] = make(map[string]bool) - } - albumsPerArtist[mf.ArtistID][mf.AlbumID] = true - } - i++ - if i%1000 == 0 { - log.Debug(fmt.Sprintf("Processed %d tracks", i), "artists", len(s.artists), "albums", len(s.albums), "songs", len(s.mediaFiles)) - } - } - - log.Debug("Finished processing tracks.", "artists", len(s.artists), "albums", len(s.albums), "songs", len(s.mediaFiles)) - - for albumId, count := range songsPerAlbum { - s.albums[albumId].SongCount = count - } - - for artistId, albums := range albumsPerArtist { - s.artists[artistId].AlbumCount = len(albums) - } - - if err := s.checksumRepo.SetData(s.newSums); err != nil { - log.Error("Error saving checksums", err) - } else { - log.Debug("Saved checksums", "total", len(s.newSums)) - } - - ignFolders := conf.Sonic.PlsIgnoreFolders - ignPatterns := strings.Split(conf.Sonic.PlsIgnoredPatterns, ";") - for _, p := range l.Playlists { - rel := plsRelation{pID: p.PlaylistPersistentID, parentPID: p.ParentPersistentID, name: unescape(p.Name)} - s.pplaylists[p.PlaylistPersistentID] = rel - fullPath := s.fullPath(p.PlaylistPersistentID) - - if s.skipPlaylist(&p, ignFolders, ignPatterns, fullPath) { - continue - } - - s.collectPlaylists(&p, fullPath) - } - log.Debug("Processed playlists", "total", len(l.Playlists)) - - return len(l.Tracks), nil -} - -func (s *ItunesScanner) MediaFiles() map[string]*model.MediaFile { - return s.mediaFiles -} -func (s *ItunesScanner) Albums() map[string]*model.Album { - return s.albums -} -func (s *ItunesScanner) Artists() map[string]*model.Artist { - return s.artists -} -func (s *ItunesScanner) Playlists() map[string]*model.Playlist { - return s.playlists -} - -func (s *ItunesScanner) skipTrack(t *itl.Track) bool { - if t.Podcast { - return true - } - - if conf.Sonic.DevDisableFileCheck { - return false - } - - if !strings.HasPrefix(t.Location, "file://") { - return true - } - - ext := filepath.Ext(t.Location) - m := mime.TypeByExtension(ext) - - return !strings.HasPrefix(m, "audio/") -} - -func (s *ItunesScanner) skipPlaylist(p *itl.Playlist, ignFolders bool, ignPatterns []string, fullPath string) bool { - // Skip all "special" iTunes playlists, and also ignored patterns - if p.Master || p.Music || p.Audiobooks || p.Movies || p.TVShows || p.Podcasts || p.ITunesU || (ignFolders && p.Folder) { - return true - } - - for _, p := range ignPatterns { - if p == "" { - continue - } - m, _ := regexp.MatchString(p, fullPath) - if m { - return true - } - } - - return false -} - -func (s *ItunesScanner) collectPlaylists(p *itl.Playlist, fullPath string) { - pl := &model.Playlist{} - pl.ID = p.PlaylistPersistentID - pl.Name = unescape(p.Name) - pl.FullPath = fullPath - pl.Tracks = make([]string, 0, len(p.PlaylistItems)) - for _, item := range p.PlaylistItems { - if mf, found := s.pmediaFiles[item.TrackID]; found { - pl.Tracks = append(pl.Tracks, mf.ID) - pl.Duration += mf.Duration - } - } - if len(pl.Tracks) > 0 { - s.playlists[pl.ID] = pl - } -} - -func (s *ItunesScanner) fullPath(pID string) string { - rel, found := s.pplaylists[pID] - if !found { - return "" - } - if rel.parentPID == "" { - return rel.name - } - return fmt.Sprintf("%s > %s", s.fullPath(rel.parentPID), rel.name) -} - -func (s *ItunesScanner) lastChangedDate(t *itl.Track) time.Time { - if s.hasChanged(t) { - return time.Now() - } - allDates := []time.Time{t.DateModified, t.PlayDateUTC} - c := time.Time{} - for _, d := range allDates { - if c.Before(d) { - c = d - } - } - return c -} - -func (s *ItunesScanner) hasChanged(t *itl.Track) bool { - id := t.PersistentID - oldSum, _ := s.checksums[id] - newSum := s.newSums[id] - return oldSum != newSum -} - -// Calc sum of stats fields (whose changes are not reflected in DataModified) -func (s *ItunesScanner) calcCheckSum(t *itl.Track) string { - id := t.PersistentID - data := fmt.Sprint(t.DateModified, t.PlayCount, t.PlayDate, t.ArtworkCount, t.Loved, t.AlbumLoved, - t.Rating, t.AlbumRating, t.SkipCount, t.SkipDate) - sum := fmt.Sprintf("%x", md5.Sum([]byte(data))) - s.newSums[id] = sum - return sum -} - -func (s *ItunesScanner) collectMediaFiles(t *itl.Track) *model.MediaFile { - mf := &model.MediaFile{} - mf.ID = t.PersistentID - mf.Album = unescape(t.Album) - mf.AlbumID = albumId(t) - mf.ArtistID = artistId(t) - mf.Title = unescape(t.Name) - mf.Artist = unescape(t.Artist) - if mf.Album == "" { - mf.Album = "[Unknown Album]" - } - if mf.Artist == "" { - mf.Artist = "[Unknown Artist]" - } - mf.AlbumArtist = unescape(t.AlbumArtist) - mf.Genre = unescape(t.Genre) - mf.Compilation = t.Compilation - mf.Starred = t.Loved - mf.Rating = t.Rating / 20 - mf.PlayCount = t.PlayCount - mf.PlayDate = t.PlayDateUTC - mf.Year = t.Year - mf.TrackNumber = t.TrackNumber - mf.DiscNumber = t.DiscNumber - if t.Size > 0 { - mf.Size = strconv.Itoa(t.Size) - } - if t.TotalTime > 0 { - mf.Duration = t.TotalTime / 1000 - } - mf.BitRate = t.BitRate - - path := extractPath(t.Location) - mf.Path = path - mf.Suffix = strings.TrimPrefix(filepath.Ext(path), ".") - - mf.CreatedAt = t.DateAdded - mf.UpdatedAt = s.lastChangedDate(t) - - if mf.UpdatedAt.After(s.lastModifiedSince) && !conf.Sonic.DevDisableFileCheck { - mf.HasCoverArt = hasCoverArt(path) - } - - s.mediaFiles[mf.ID] = mf - s.pmediaFiles[t.TrackID] = mf - - return mf -} - -func (s *ItunesScanner) collectAlbums(t *itl.Track, mf *model.MediaFile, ar *model.Artist) *model.Album { - id := albumId(t) - _, found := s.albums[id] - if !found { - s.albums[id] = &model.Album{} - } - - al := s.albums[id] - al.ID = id - al.ArtistID = ar.ID - al.Name = mf.Album - al.Year = t.Year - al.Compilation = t.Compilation - al.Starred = t.AlbumLoved - al.Rating = t.AlbumRating / 20 - al.PlayCount += t.PlayCount - al.Genre = mf.Genre - al.Artist = mf.Artist - al.AlbumArtist = ar.Name - if al.Name == "" { - al.Name = "[Unknown Album]" - } - if al.Artist == "" { - al.Artist = "[Unknown Artist]" - } - al.Duration += mf.Duration - - if mf.HasCoverArt { - al.CoverArtId = mf.ID - al.CoverArtPath = mf.Path - } - - if al.PlayDate.IsZero() || t.PlayDateUTC.After(al.PlayDate) { - al.PlayDate = t.PlayDateUTC - } - if al.CreatedAt.IsZero() || t.DateAdded.Before(al.CreatedAt) { - al.CreatedAt = t.DateAdded - } - trackUpdate := s.lastChangedDate(t) - if al.UpdatedAt.IsZero() || trackUpdate.After(al.UpdatedAt) { - al.UpdatedAt = trackUpdate - } - - return al -} - -func (s *ItunesScanner) collectArtists(t *itl.Track) *model.Artist { - id := artistId(t) - _, found := s.artists[id] - if !found { - s.artists[id] = &model.Artist{} - } - ar := s.artists[id] - ar.ID = id - ar.Name = unescape(realArtistName(t)) - if ar.Name == "" { - ar.Name = "[Unknown Artist]" - } - - return ar -} - -func albumId(t *itl.Track) string { - s := strings.ToLower(fmt.Sprintf("%s\\%s", realArtistName(t), t.Album)) - return fmt.Sprintf("%x", md5.Sum([]byte(s))) -} - -func artistId(t *itl.Track) string { - return fmt.Sprintf("%x", md5.Sum([]byte(strings.ToLower(realArtistName(t))))) -} - -func hasCoverArt(path string) bool { - defer func() { - if r := recover(); r != nil { - log.Error("Panic reading tag", "path", path, "error", r) - } - }() - - if _, err := os.Stat(path); err == nil { - f, err := os.Open(path) - if err != nil { - log.Warn("Error opening file", "path", path, err) - return false - } - defer f.Close() - - m, err := tag.ReadFrom(f) - if err != nil { - log.Warn("Error reading tag from file", "path", path, err) - return false - } - return m.Picture() != nil - } - //log.Warn("File not found", "path", path) - return false -} - -func unescape(str string) string { - return html.UnescapeString(str) -} - -func extractPath(loc string) string { - path := strings.Replace(loc, "+", "%2B", -1) - path, _ = url.QueryUnescape(path) - path = html.UnescapeString(path) - return strings.TrimPrefix(path, "file://") -} - -func realArtistName(t *itl.Track) string { - switch { - case t.Compilation: - return "Various Artists" - case t.AlbumArtist != "": - return t.AlbumArtist - } - - return t.Artist -} - -var _ Scanner = (*ItunesScanner)(nil) diff --git a/scanner_legacy/itunes_scanner_test.go b/scanner_legacy/itunes_scanner_test.go deleted file mode 100644 index b7de75007..000000000 --- a/scanner_legacy/itunes_scanner_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package scanner_legacy - -import ( - "testing" - - . "github.com/smartystreets/goconvey/convey" -) - -func TestExtractLocation(t *testing.T) { - - Convey("Given a path with a plus (+) signal", t, func() { - location := "file:///Users/deluan/Music/iTunes%201/iTunes%20Media/Music/Chance/Six%20Through%20Ten/03%20Forgive+Forget.m4a" - - Convey("When I decode it", func() { - path := extractPath(location) - - Convey("I get the correct path", func() { - So(path, ShouldEqual, "/Users/deluan/Music/iTunes 1/iTunes Media/Music/Chance/Six Through Ten/03 Forgive+Forget.m4a") - }) - - }) - - }) - -} diff --git a/scanner_legacy/wire_providers.go b/scanner_legacy/wire_providers.go deleted file mode 100644 index 107bd7e44..000000000 --- a/scanner_legacy/wire_providers.go +++ /dev/null @@ -1,9 +0,0 @@ -package scanner_legacy - -import "github.com/google/wire" - -var Set = wire.NewSet( - NewImporter, - NewItunesScanner, - wire.Bind(new(Scanner), new(*ItunesScanner)), -) diff --git a/server/app.go b/server/app.go index 7ffc2f9b5..b4668c364 100644 --- a/server/app.go +++ b/server/app.go @@ -10,7 +10,6 @@ import ( "github.com/cloudsonic/sonic-server/conf" "github.com/cloudsonic/sonic-server/log" "github.com/cloudsonic/sonic-server/scanner" - "github.com/cloudsonic/sonic-server/scanner_legacy" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/go-chi/cors" @@ -19,25 +18,18 @@ import ( const Version = "0.2" type Server struct { - Importer *scanner_legacy.Importer - Scanner *scanner.Scanner - router *chi.Mux + Scanner *scanner.Scanner + router *chi.Mux } -func New(importer *scanner_legacy.Importer, scanner *scanner.Scanner) *Server { - a := &Server{Importer: importer, Scanner: scanner} +func New(scanner *scanner.Scanner) *Server { + a := &Server{Scanner: scanner} if !conf.Sonic.DevDisableBanner { showBanner(Version) } initMimeTypes() a.initRoutes() - if conf.Sonic.DevUseFileScanner { - log.Info("Using Folder Scanner", "folder", conf.Sonic.MusicFolder) - a.initScanner() - } else { - log.Info("Using iTunes Importer", "xml", conf.Sonic.MusicFolder) - a.initImporter() - } + a.initScanner() return a } @@ -89,22 +81,6 @@ func (a *Server) initScanner() { }() } -func (a *Server) initImporter() { - go func() { - first := true - for { - select { - case <-time.After(5 * time.Second): - if first { - log.Info("Started iTunes scanner", "xml", conf.Sonic.MusicFolder) - first = false - } - a.Importer.CheckForUpdates(false) - } - } - }() -} - func FileServer(r chi.Router, path string, root http.FileSystem) { if strings.ContainsAny(path, "{}*") { panic("FileServer does not permit URL parameters.") diff --git a/wire_gen.go b/wire_gen.go index 982aeac23..989084db6 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -8,10 +8,8 @@ package main import ( "github.com/cloudsonic/sonic-server/api" "github.com/cloudsonic/sonic-server/engine" - "github.com/cloudsonic/sonic-server/itunesbridge" "github.com/cloudsonic/sonic-server/persistence" "github.com/cloudsonic/sonic-server/scanner" - "github.com/cloudsonic/sonic-server/scanner_legacy" "github.com/cloudsonic/sonic-server/server" "github.com/google/wire" ) @@ -19,41 +17,26 @@ import ( // Injectors from wire_injectors.go: func CreateApp(musicFolder string) *server.Server { - checksumRepository := persistence.NewCheckSumRepository() - itunesScanner := scanner_legacy.NewItunesScanner(checksumRepository) - mediaFileRepository := persistence.NewMediaFileRepository() - albumRepository := persistence.NewAlbumRepository() - artistRepository := persistence.NewArtistRepository() - playlistRepository := persistence.NewPlaylistRepository() - propertyRepository := persistence.NewPropertyRepository() - importer := scanner_legacy.NewImporter(musicFolder, itunesScanner, mediaFileRepository, albumRepository, artistRepository, playlistRepository, propertyRepository) - mediaFolderRepository := persistence.NewMediaFolderRepository() - scannerScanner := scanner.New(mediaFileRepository, albumRepository, artistRepository, playlistRepository, mediaFolderRepository, propertyRepository) - serverServer := server.New(importer, scannerScanner) + dataStore := persistence.New() + scannerScanner := scanner.New(dataStore) + serverServer := server.New(scannerScanner) return serverServer } func CreateSubsonicAPIRouter() *api.Router { - propertyRepository := persistence.NewPropertyRepository() - mediaFolderRepository := persistence.NewMediaFolderRepository() - artistRepository := persistence.NewArtistRepository() - albumRepository := persistence.NewAlbumRepository() - mediaFileRepository := persistence.NewMediaFileRepository() - genreRepository := persistence.NewGenreRepository() - browser := engine.NewBrowser(propertyRepository, mediaFolderRepository, artistRepository, albumRepository, mediaFileRepository, genreRepository) - cover := engine.NewCover(mediaFileRepository, albumRepository) + dataStore := persistence.New() + browser := engine.NewBrowser(dataStore) + cover := engine.NewCover(dataStore) nowPlayingRepository := engine.NewNowPlayingRepository() - listGenerator := engine.NewListGenerator(artistRepository, albumRepository, mediaFileRepository, nowPlayingRepository) - itunesControl := itunesbridge.NewItunesControl() - playlistRepository := persistence.NewPlaylistRepository() - playlists := engine.NewPlaylists(itunesControl, playlistRepository, mediaFileRepository) - ratings := engine.NewRatings(itunesControl, mediaFileRepository, albumRepository, artistRepository) - scrobbler := engine.NewScrobbler(itunesControl, mediaFileRepository, albumRepository, nowPlayingRepository) - search := engine.NewSearch(artistRepository, albumRepository, mediaFileRepository) + listGenerator := engine.NewListGenerator(dataStore, nowPlayingRepository) + playlists := engine.NewPlaylists(dataStore) + ratings := engine.NewRatings(dataStore) + scrobbler := engine.NewScrobbler(dataStore, nowPlayingRepository) + search := engine.NewSearch(dataStore) router := api.NewRouter(browser, cover, listGenerator, playlists, ratings, scrobbler, search) return router } // wire_injectors.go: -var allProviders = wire.NewSet(itunesbridge.NewItunesControl, engine.Set, scanner_legacy.Set, scanner.New, api.NewRouter, persistence.Set) +var allProviders = wire.NewSet(engine.Set, scanner.New, api.NewRouter, persistence.Set) diff --git a/wire_injectors.go b/wire_injectors.go index 2ae17419a..812899fce 100644 --- a/wire_injectors.go +++ b/wire_injectors.go @@ -5,18 +5,14 @@ package main import ( "github.com/cloudsonic/sonic-server/api" "github.com/cloudsonic/sonic-server/engine" - "github.com/cloudsonic/sonic-server/itunesbridge" "github.com/cloudsonic/sonic-server/persistence" "github.com/cloudsonic/sonic-server/scanner" - "github.com/cloudsonic/sonic-server/scanner_legacy" "github.com/cloudsonic/sonic-server/server" "github.com/google/wire" ) var allProviders = wire.NewSet( - itunesbridge.NewItunesControl, engine.Set, - scanner_legacy.Set, scanner.New, api.NewRouter, persistence.Set,