Add denormalized list of artist_ids to album, to speed-up artist's albums queries

This will be removed once we have a proper many-to-many relationship between album and artist

db/migration/20201213124814_add_all_artist_ids_to_album.go

@@ -0,0 +1,62 @@
+package migration
+
+import (
+	"database/sql"
+
+	"github.com/deluan/navidrome/log"
+	"github.com/deluan/navidrome/utils"
+	"github.com/pressly/goose"
+)
+
+func init() {
+	goose.AddMigration(Up20201213124814, Down20201213124814)
+}
+
+func Up20201213124814(tx *sql.Tx) error {
+	_, err := tx.Exec(`
+alter table album
+	add all_artist_ids varchar;
+
+create index if not exists album_all_artist_ids
+	on album (all_artist_ids);
+`)
+	if err != nil {
+		return err
+	}
+
+	return updateAlbums20201213124814(tx)
+}
+
+func updateAlbums20201213124814(tx *sql.Tx) error {
+	rows, err := tx.Query(`
+select a.id, a.name, a.artist_id, a.album_artist_id, group_concat(mf.artist_id, ' ') 
+       from album a left join media_file mf on a.id = mf.album_id group by a.id
+   `)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	stmt, err := tx.Prepare("update album set all_artist_ids = ? where id = ?")
+	if err != nil {
+		return err
+	}
+
+	var id, name, artistId, albumArtistId, songArtistIds string
+	for rows.Next() {
+		err := rows.Scan(&id, &name, &artistId, &albumArtistId, &songArtistIds)
+		if err != nil {
+			return err
+		}
+		all := utils.SanitizeStrings(artistId, albumArtistId, songArtistIds)
+		_, err = stmt.Exec(all, id)
+		if err != nil {
+			log.Error("Error setting album's artist_ids", "album", name, "albumId", id, err)
+		}
+	}
+	return rows.Err()
+}
+
+func Down20201213124814(tx *sql.Tx) error {
+	return nil
+}

model/album.go

@@ -13,12 +13,14 @@ type Album struct {
 	Artist               string    `json:"artist"`
 	AlbumArtistID        string    `json:"albumArtistId" orm:"column(album_artist_id)"`
 	AlbumArtist          string    `json:"albumArtist"`
+	AllArtistIDs         string    `json:"allArtistIds"  orm:"column(all_artist_ids)"`
 	MaxYear              int       `json:"maxYear"`
 	MinYear              int       `json:"minYear"`
 	Compilation          bool      `json:"compilation"`
 	Comment              string    `json:"comment"`
 	SongCount            int       `json:"songCount"`
 	Duration             float32   `json:"duration"`
+	Size                 int64     `json:"size"`
 	Genre                string    `json:"genre"`
 	FullText             string    `json:"fullText"`
 	SortAlbumName        string    `json:"sortAlbumName"`
@@ -33,7 +35,6 @@ type Album struct {
 	MbzAlbumComment      string    `json:"mbzAlbumComment"`
 	CreatedAt            time.Time `json:"createdAt"`
 	UpdatedAt            time.Time `json:"updatedAt"`
-	Size                 int64     `json:"size"`
 }
 
 type Albums []Album

persistence/album_repository.go

@@ -2,6 +2,7 @@ package persistence
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"sort"
@@ -63,13 +64,7 @@ func yearFilter(field string, value interface{}) Sqlizer {
 }
 
 func artistFilter(field string, value interface{}) Sqlizer {
-	return exists("media_file", And{
+	return Like{"all_artist_ids": fmt.Sprintf("%%%s%%", value)}
-		ConcatExpr("album_id=album.id"),
-		Or{
-			Eq{"artist_id": value},
-			Eq{"album_artist_id": value},
-		},
-	})
 }
 
 func (r *albumRepository) CountAll(options ...model.QueryOptions) (int64, error) {
@@ -153,6 +148,7 @@ func (r *albumRepository) refresh(ids ...string) error {
 		model.Album
 		CurrentId     string
 		SongArtists   string
+		SongArtistIds string
 		Years         string
 		DiscSubtitles string
 		Comments      string
@@ -167,7 +163,9 @@ func (r *albumRepository) refresh(ids ...string) error {
 		f.catalog_num, f.compilation, f.genre, max(f.year) as max_year, sum(f.duration) as duration, 
 		count(f.id) as song_count, a.id as current_id, 
 		group_concat(f.disc_subtitle, ' ') as disc_subtitles,
-		group_concat(f.artist, ' ') as song_artists, group_concat(f.year, ' ') as years,
+		group_concat(f.artist, ' ') as song_artists, 
+		group_concat(f.artist_id, ' ') as song_artist_ids, 
+		group_concat(f.year, ' ') as years,
 		sum(f.size) as size`).
 		From("media_file f").
 		LeftJoin("album a on f.album_id = a.id").
@@ -222,6 +220,7 @@ func (r *albumRepository) refresh(ids ...string) error {
 			toInsert++
 			al.CreatedAt = time.Now()
 		}
+		al.AllArtistIDs = utils.SanitizeStrings(al.SongArtistIds, al.AlbumArtistID, al.ArtistID)
 		al.FullText = getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists,
 			al.SortAlbumName, al.SortArtistName, al.SortAlbumArtistName, al.DiscSubtitles)
 		_, err := r.put(al.ID, al.Album)

persistence/sql_search.go

@@ -1,42 +1,18 @@
 package persistence
 
 import (
-	"regexp"
-	"sort"
 	"strings"
 
 	. "github.com/Masterminds/squirrel"
 	"github.com/deluan/navidrome/conf"
-	"github.com/kennygrant/sanitize"
+	"github.com/deluan/navidrome/utils"
 )
 
-var quotesRegex = regexp.MustCompile("[“”‘’'\"\\[\\(\\{\\]\\)\\}]")
-
 func getFullText(text ...string) string {
-	fullText := sanitizeStrings(text...)
+	fullText := utils.SanitizeStrings(text...)
 	return " " + fullText
 }
 
-func sanitizeStrings(text ...string) string {
-	sanitizedText := strings.Builder{}
-	for _, txt := range text {
-		sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ")
-	}
-	words := make(map[string]struct{})
-	for _, w := range strings.Fields(sanitizedText.String()) {
-		words[w] = struct{}{}
-	}
-	var fullText []string
-	for w := range words {
-		w = quotesRegex.ReplaceAllString(w, "")
-		if w != "" {
-			fullText = append(fullText, w)
-		}
-	}
-	sort.Strings(fullText)
-	return strings.Join(fullText, " ")
-}
-
 func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, orderBys ...string) error {
 	q = strings.TrimSpace(q)
 	q = strings.TrimSuffix(q, "*")
@@ -59,7 +35,7 @@ func fullTextExpr(value string) Sqlizer {
 	if !conf.Server.SearchFullString {
 		sep = " "
 	}
-	q := sanitizeStrings(value)
+	q := utils.SanitizeStrings(value)
 	parts := strings.Split(q, " ")
 	filters := And{}
 	for _, part := range parts {

persistence/sql_search_test.go

@@ -7,28 +7,8 @@ import (
 
 var _ = Describe("sqlRepository", func() {
 	Describe("getFullText", func() {
-		It("returns all lowercase chars", func() {
+		It("prefixes with a space", func() {
-			Expect(getFullText("Some Text")).To(Equal(" some text"))
+			Expect(getFullText("legiao urbana")).To(Equal(" legiao urbana"))
-		})
-
-		It("removes accents", func() {
-			Expect(getFullText("Quintão")).To(Equal(" quintao"))
-		})
-
-		It("remove extra spaces", func() {
-			Expect(getFullText(" some  text  ")).To(Equal(" some text"))
-		})
-
-		It("remove duplicated words", func() {
-			Expect(getFullText("legião urbana urbana legiÃo")).To(Equal(" legiao urbana"))
-		})
-
-		It("remove symbols", func() {
-			Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms"))
-		})
-
-		It("remove opening brackets", func() {
-			Expect(getFullText("[Five Years]")).To(Equal(" five years"))
 		})
 	})
 })

utils/sanitize_strings.go

@@ -0,0 +1,31 @@
+package utils
+
+import (
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/kennygrant/sanitize"
+)
+
+var quotesRegex = regexp.MustCompile("[“”‘’'\"\\[\\(\\{\\]\\)\\}]")
+
+func SanitizeStrings(text ...string) string {
+	sanitizedText := strings.Builder{}
+	for _, txt := range text {
+		sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ")
+	}
+	words := make(map[string]struct{})
+	for _, w := range strings.Fields(sanitizedText.String()) {
+		words[w] = struct{}{}
+	}
+	var fullText []string
+	for w := range words {
+		w = quotesRegex.ReplaceAllString(w, "")
+		if w != "" {
+			fullText = append(fullText, w)
+		}
+	}
+	sort.Strings(fullText)
+	return strings.Join(fullText, " ")
+}

utils/sanitize_strings_test.go

@@ -0,0 +1,32 @@
+package utils
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("SanitizeStrings", func() {
+	It("returns all lowercase chars", func() {
+		Expect(SanitizeStrings("Some Text")).To(Equal("some text"))
+	})
+
+	It("removes accents", func() {
+		Expect(SanitizeStrings("Quintão")).To(Equal("quintao"))
+	})
+
+	It("remove extra spaces", func() {
+		Expect(SanitizeStrings(" some  text  ")).To(Equal("some text"))
+	})
+
+	It("remove duplicated words", func() {
+		Expect(SanitizeStrings("legião urbana urbana legiÃo")).To(Equal("legiao urbana"))
+	})
+
+	It("remove symbols", func() {
+		Expect(SanitizeStrings("Tom’s Diner ' “40” ‘A’")).To(Equal("40 a diner toms"))
+	})
+
+	It("remove opening brackets", func() {
+		Expect(SanitizeStrings("[Five Years]")).To(Equal("five years"))
+	})
+})