From 4f90fa9924630a9dc6b279eaf8b0165c1452923b Mon Sep 17 00:00:00 2001 From: Deluan Date: Sun, 13 Dec 2020 14:05:48 -0500 Subject: [PATCH] Add denormalized list of artist_ids to album, to speed-up artist's albums queries This will be removed once we have a proper many-to-many relationship between album and artist --- ...01213124814_add_all_artist_ids_to_album.go | 62 +++++++++++++++++++ model/album.go | 3 +- persistence/album_repository.go | 15 +++-- persistence/sql_search.go | 30 +-------- persistence/sql_search_test.go | 24 +------ utils/sanitize_strings.go | 31 ++++++++++ utils/sanitize_strings_test.go | 32 ++++++++++ 7 files changed, 139 insertions(+), 58 deletions(-) create mode 100644 db/migration/20201213124814_add_all_artist_ids_to_album.go create mode 100644 utils/sanitize_strings.go create mode 100644 utils/sanitize_strings_test.go diff --git a/db/migration/20201213124814_add_all_artist_ids_to_album.go b/db/migration/20201213124814_add_all_artist_ids_to_album.go new file mode 100644 index 000000000..b43c37d5d --- /dev/null +++ b/db/migration/20201213124814_add_all_artist_ids_to_album.go @@ -0,0 +1,62 @@ +package migration + +import ( + "database/sql" + + "github.com/deluan/navidrome/log" + "github.com/deluan/navidrome/utils" + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(Up20201213124814, Down20201213124814) +} + +func Up20201213124814(tx *sql.Tx) error { + _, err := tx.Exec(` +alter table album + add all_artist_ids varchar; + +create index if not exists album_all_artist_ids + on album (all_artist_ids); +`) + if err != nil { + return err + } + + return updateAlbums20201213124814(tx) +} + +func updateAlbums20201213124814(tx *sql.Tx) error { + rows, err := tx.Query(` +select a.id, a.name, a.artist_id, a.album_artist_id, group_concat(mf.artist_id, ' ') + from album a left join media_file mf on a.id = mf.album_id group by a.id + `) + if err != nil { + return err + } + defer rows.Close() + + stmt, err := tx.Prepare("update album set all_artist_ids = ? where id = ?") + if err != nil { + return err + } + + var id, name, artistId, albumArtistId, songArtistIds string + for rows.Next() { + err := rows.Scan(&id, &name, &artistId, &albumArtistId, &songArtistIds) + if err != nil { + return err + } + all := utils.SanitizeStrings(artistId, albumArtistId, songArtistIds) + _, err = stmt.Exec(all, id) + if err != nil { + log.Error("Error setting album's artist_ids", "album", name, "albumId", id, err) + } + } + return rows.Err() +} + +func Down20201213124814(tx *sql.Tx) error { + return nil +} diff --git a/model/album.go b/model/album.go index 8a712b901..7b32a6265 100644 --- a/model/album.go +++ b/model/album.go @@ -13,12 +13,14 @@ type Album struct { Artist string `json:"artist"` AlbumArtistID string `json:"albumArtistId" orm:"column(album_artist_id)"` AlbumArtist string `json:"albumArtist"` + AllArtistIDs string `json:"allArtistIds" orm:"column(all_artist_ids)"` MaxYear int `json:"maxYear"` MinYear int `json:"minYear"` Compilation bool `json:"compilation"` Comment string `json:"comment"` SongCount int `json:"songCount"` Duration float32 `json:"duration"` + Size int64 `json:"size"` Genre string `json:"genre"` FullText string `json:"fullText"` SortAlbumName string `json:"sortAlbumName"` @@ -33,7 +35,6 @@ type Album struct { MbzAlbumComment string `json:"mbzAlbumComment"` CreatedAt time.Time `json:"createdAt"` UpdatedAt time.Time `json:"updatedAt"` - Size int64 `json:"size"` } type Albums []Album diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 5268049b0..c2179e6f2 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -2,6 +2,7 @@ package persistence import ( "context" + "fmt" "os" "path/filepath" "sort" @@ -63,13 +64,7 @@ func yearFilter(field string, value interface{}) Sqlizer { } func artistFilter(field string, value interface{}) Sqlizer { - return exists("media_file", And{ - ConcatExpr("album_id=album.id"), - Or{ - Eq{"artist_id": value}, - Eq{"album_artist_id": value}, - }, - }) + return Like{"all_artist_ids": fmt.Sprintf("%%%s%%", value)} } func (r *albumRepository) CountAll(options ...model.QueryOptions) (int64, error) { @@ -153,6 +148,7 @@ func (r *albumRepository) refresh(ids ...string) error { model.Album CurrentId string SongArtists string + SongArtistIds string Years string DiscSubtitles string Comments string @@ -167,7 +163,9 @@ func (r *albumRepository) refresh(ids ...string) error { f.catalog_num, f.compilation, f.genre, max(f.year) as max_year, sum(f.duration) as duration, count(f.id) as song_count, a.id as current_id, group_concat(f.disc_subtitle, ' ') as disc_subtitles, - group_concat(f.artist, ' ') as song_artists, group_concat(f.year, ' ') as years, + group_concat(f.artist, ' ') as song_artists, + group_concat(f.artist_id, ' ') as song_artist_ids, + group_concat(f.year, ' ') as years, sum(f.size) as size`). From("media_file f"). LeftJoin("album a on f.album_id = a.id"). @@ -222,6 +220,7 @@ func (r *albumRepository) refresh(ids ...string) error { toInsert++ al.CreatedAt = time.Now() } + al.AllArtistIDs = utils.SanitizeStrings(al.SongArtistIds, al.AlbumArtistID, al.ArtistID) al.FullText = getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists, al.SortAlbumName, al.SortArtistName, al.SortAlbumArtistName, al.DiscSubtitles) _, err := r.put(al.ID, al.Album) diff --git a/persistence/sql_search.go b/persistence/sql_search.go index 21eaa39b2..c5b829c43 100644 --- a/persistence/sql_search.go +++ b/persistence/sql_search.go @@ -1,42 +1,18 @@ package persistence import ( - "regexp" - "sort" "strings" . "github.com/Masterminds/squirrel" "github.com/deluan/navidrome/conf" - "github.com/kennygrant/sanitize" + "github.com/deluan/navidrome/utils" ) -var quotesRegex = regexp.MustCompile("[“”‘’'\"\\[\\(\\{\\]\\)\\}]") - func getFullText(text ...string) string { - fullText := sanitizeStrings(text...) + fullText := utils.SanitizeStrings(text...) return " " + fullText } -func sanitizeStrings(text ...string) string { - sanitizedText := strings.Builder{} - for _, txt := range text { - sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ") - } - words := make(map[string]struct{}) - for _, w := range strings.Fields(sanitizedText.String()) { - words[w] = struct{}{} - } - var fullText []string - for w := range words { - w = quotesRegex.ReplaceAllString(w, "") - if w != "" { - fullText = append(fullText, w) - } - } - sort.Strings(fullText) - return strings.Join(fullText, " ") -} - func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, orderBys ...string) error { q = strings.TrimSpace(q) q = strings.TrimSuffix(q, "*") @@ -59,7 +35,7 @@ func fullTextExpr(value string) Sqlizer { if !conf.Server.SearchFullString { sep = " " } - q := sanitizeStrings(value) + q := utils.SanitizeStrings(value) parts := strings.Split(q, " ") filters := And{} for _, part := range parts { diff --git a/persistence/sql_search_test.go b/persistence/sql_search_test.go index 231528aaf..60d4df755 100644 --- a/persistence/sql_search_test.go +++ b/persistence/sql_search_test.go @@ -7,28 +7,8 @@ import ( var _ = Describe("sqlRepository", func() { Describe("getFullText", func() { - It("returns all lowercase chars", func() { - Expect(getFullText("Some Text")).To(Equal(" some text")) - }) - - It("removes accents", func() { - Expect(getFullText("Quintão")).To(Equal(" quintao")) - }) - - It("remove extra spaces", func() { - Expect(getFullText(" some text ")).To(Equal(" some text")) - }) - - It("remove duplicated words", func() { - Expect(getFullText("legião urbana urbana legiÃo")).To(Equal(" legiao urbana")) - }) - - It("remove symbols", func() { - Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms")) - }) - - It("remove opening brackets", func() { - Expect(getFullText("[Five Years]")).To(Equal(" five years")) + It("prefixes with a space", func() { + Expect(getFullText("legiao urbana")).To(Equal(" legiao urbana")) }) }) }) diff --git a/utils/sanitize_strings.go b/utils/sanitize_strings.go new file mode 100644 index 000000000..e64c5654e --- /dev/null +++ b/utils/sanitize_strings.go @@ -0,0 +1,31 @@ +package utils + +import ( + "regexp" + "sort" + "strings" + + "github.com/kennygrant/sanitize" +) + +var quotesRegex = regexp.MustCompile("[“”‘’'\"\\[\\(\\{\\]\\)\\}]") + +func SanitizeStrings(text ...string) string { + sanitizedText := strings.Builder{} + for _, txt := range text { + sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ") + } + words := make(map[string]struct{}) + for _, w := range strings.Fields(sanitizedText.String()) { + words[w] = struct{}{} + } + var fullText []string + for w := range words { + w = quotesRegex.ReplaceAllString(w, "") + if w != "" { + fullText = append(fullText, w) + } + } + sort.Strings(fullText) + return strings.Join(fullText, " ") +} diff --git a/utils/sanitize_strings_test.go b/utils/sanitize_strings_test.go new file mode 100644 index 000000000..daddc0753 --- /dev/null +++ b/utils/sanitize_strings_test.go @@ -0,0 +1,32 @@ +package utils + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("SanitizeStrings", func() { + It("returns all lowercase chars", func() { + Expect(SanitizeStrings("Some Text")).To(Equal("some text")) + }) + + It("removes accents", func() { + Expect(SanitizeStrings("Quintão")).To(Equal("quintao")) + }) + + It("remove extra spaces", func() { + Expect(SanitizeStrings(" some text ")).To(Equal("some text")) + }) + + It("remove duplicated words", func() { + Expect(SanitizeStrings("legião urbana urbana legiÃo")).To(Equal("legiao urbana")) + }) + + It("remove symbols", func() { + Expect(SanitizeStrings("Tom’s Diner ' “40” ‘A’")).To(Equal("40 a diner toms")) + }) + + It("remove opening brackets", func() { + Expect(SanitizeStrings("[Five Years]")).To(Equal("five years")) + }) +})