diff --git a/db/migration/20200419222708_reindex_to_change_full_text_search.go b/db/migration/20200419222708_reindex_to_change_full_text_search.go new file mode 100644 index 000000000..b92641be4 --- /dev/null +++ b/db/migration/20200419222708_reindex_to_change_full_text_search.go @@ -0,0 +1,19 @@ +package migration + +import ( + "database/sql" + "github.com/pressly/goose" +) + +func init() { + goose.AddMigration(Up20200419222708, Down20200419222708) +} + +func Up20200419222708(tx *sql.Tx) error { + notice(tx, "A full rescan will be performed to change the search behaviour") + return forceFullRescan(tx) +} + +func Down20200419222708(tx *sql.Tx) error { + return nil +} diff --git a/persistence/album_repository.go b/persistence/album_repository.go index 098885ecc..0d62843d8 100644 --- a/persistence/album_repository.go +++ b/persistence/album_repository.go @@ -148,7 +148,7 @@ func (r *albumRepository) Refresh(ids ...string) error { toInsert++ al.CreatedAt = time.Now() } - al.FullText = r.getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists) + al.FullText = getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists) _, err := r.put(al.ID, al.Album) if err != nil { return err diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go index 1236b57b5..e057344f2 100644 --- a/persistence/album_repository_test.go +++ b/persistence/album_repository_test.go @@ -20,7 +20,7 @@ var _ = Describe("AlbumRepository", func() { Describe("Get", func() { It("returns an existent album", func() { - Expect(repo.Get("3")).To(Equal(&albumRadioactivity)) + Expect(repo.Get("103")).To(Equal(&albumRadioactivity)) }) It("returns ErrNotFound when the album does not exist", func() { _, err := repo.Get("666") diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go index 7496d0f53..7b161b241 100644 --- a/persistence/artist_repository.go +++ b/persistence/artist_repository.go @@ -56,7 +56,7 @@ func (r *artistRepository) getIndexKey(a *model.Artist) string { } func (r *artistRepository) Put(a *model.Artist) error { - a.FullText = r.getFullText(a.Name) + a.FullText = getFullText(a.Name) _, err := r.put(a.ID, a) return err } diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go index f6c29f4d4..a725e2f68 100644 --- a/persistence/mediafile_repository.go +++ b/persistence/mediafile_repository.go @@ -41,7 +41,7 @@ func (r mediaFileRepository) Exists(id string) (bool, error) { } func (r mediaFileRepository) Put(m *model.MediaFile) error { - m.FullText = r.getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist) + m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist) _, err := r.put(m.ID, m) return err } diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go index 20af87f46..ea09e5a64 100644 --- a/persistence/mediafile_repository_test.go +++ b/persistence/mediafile_repository_test.go @@ -21,7 +21,7 @@ var _ = Describe("MediaRepository", func() { }) It("gets mediafile from the DB", func() { - Expect(mr.Get("4")).To(Equal(&songAntenna)) + Expect(mr.Get("1004")).To(Equal(&songAntenna)) }) It("returns ErrNotFound", func() { @@ -39,7 +39,7 @@ var _ = Describe("MediaRepository", func() { }) It("find mediafiles by album", func() { - Expect(mr.FindByAlbum("3")).To(Equal(model.MediaFiles{ + Expect(mr.FindByAlbum("103")).To(Equal(model.MediaFiles{ songRadioactivity, songAntenna, })) diff --git a/persistence/persistence_suite_test.go b/persistence/persistence_suite_test.go index fc2414124..4e6b8703b 100644 --- a/persistence/persistence_suite_test.go +++ b/persistence/persistence_suite_test.go @@ -31,8 +31,8 @@ func TestPersistence(t *testing.T) { } var ( - artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: "kraftwerk"} - artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: "beatles the"} + artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: " kraftwerk"} + artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: " beatles the"} testArtists = model.Artists{ artistKraftwerk, artistBeatles, @@ -40,9 +40,9 @@ var ( ) var ( - albumSgtPeppers = model.Album{ID: "1", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: "beatles peppers sgt the"} - albumAbbeyRoad = model.Album{ID: "2", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: "abbey beatles road the"} - albumRadioactivity = model.Album{ID: "3", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: "kraftwerk radioactivity"} + albumSgtPeppers = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the"} + albumAbbeyRoad = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the"} + albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity"} testAlbums = model.Albums{ albumSgtPeppers, albumAbbeyRoad, @@ -51,10 +51,10 @@ var ( ) var ( - songDayInALife = model.MediaFile{ID: "1", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "1", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: "a beatles day in life peppers sgt the"} - songComeTogether = model.MediaFile{ID: "2", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "2", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: "abbey beatles come road the together"} - songRadioactivity = model.MediaFile{ID: "3", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: "kraftwerk radioactivity"} - songAntenna = model.MediaFile{ID: "4", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: "antenna kraftwerk"} + songDayInALife = model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"} + songComeTogether = model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"} + songRadioactivity = model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"} + songAntenna = model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: " antenna kraftwerk"} testSongs = model.MediaFiles{ songDayInALife, songComeTogether, @@ -70,9 +70,9 @@ var ( Comment: "No Comments", Owner: "userid", Public: true, - Tracks: model.MediaFiles{{ID: "1"}, {ID: "3"}}, + Tracks: model.MediaFiles{{ID: "1001"}, {ID: "1003"}}, } - plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "4"}}} + plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "1004"}}} testPlaylists = model.Playlists{plsBest, plsCool} ) diff --git a/persistence/playlist_repository_test.go b/persistence/playlist_repository_test.go index 85eb96a3e..271810146 100644 --- a/persistence/playlist_repository_test.go +++ b/persistence/playlist_repository_test.go @@ -63,19 +63,19 @@ var _ = Describe("PlaylistRepository", func() { Describe("Put/Exists/Delete", func() { var newPls model.Playlist BeforeEach(func() { - newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "4"}, {ID: "3"}}} + newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "1004"}, {ID: "1003"}}} }) It("saves the playlist to the DB", func() { Expect(repo.Put(&newPls)).To(BeNil()) }) It("adds repeated songs to a playlist and keeps the order", func() { - newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "4"}) + newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "1004"}) Expect(repo.Put(&newPls)).To(BeNil()) saved, _ := repo.Get("22") Expect(saved.Tracks).To(HaveLen(3)) - Expect(saved.Tracks[0].ID).To(Equal("4")) - Expect(saved.Tracks[1].ID).To(Equal("3")) - Expect(saved.Tracks[2].ID).To(Equal("4")) + Expect(saved.Tracks[0].ID).To(Equal("1004")) + Expect(saved.Tracks[1].ID).To(Equal("1003")) + Expect(saved.Tracks[2].ID).To(Equal("1004")) }) It("returns the newly created playlist", func() { Expect(repo.Exists("22")).To(BeTrue()) diff --git a/persistence/sql_restful.go b/persistence/sql_restful.go index eb56b5cfe..0269c29df 100644 --- a/persistence/sql_restful.go +++ b/persistence/sql_restful.go @@ -7,7 +7,6 @@ import ( . "github.com/Masterminds/squirrel" "github.com/deluan/navidrome/model" "github.com/deluan/rest" - "github.com/kennygrant/sanitize" ) type filterFunc = func(field string, value interface{}) Sqlizer @@ -59,15 +58,11 @@ func booleanFilter(field string, value interface{}) Sqlizer { } func fullTextFilter(field string, value interface{}) Sqlizer { - q := value.(string) - q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*")))) + q := sanitizeStrings(value.(string)) parts := strings.Split(q, " ") filters := And{} for _, part := range parts { - filters = append(filters, Or{ - Like{"full_text": part + "%"}, - Like{"full_text": "%" + part + "%"}, - }) + filters = append(filters, Like{"full_text": "% " + part + "%"}) } return filters } diff --git a/persistence/sql_search.go b/persistence/sql_search.go index fb7f52688..2359e5363 100644 --- a/persistence/sql_search.go +++ b/persistence/sql_search.go @@ -1,6 +1,7 @@ package persistence import ( + "regexp" "sort" "strings" @@ -8,7 +9,14 @@ import ( "github.com/kennygrant/sanitize" ) -func (r sqlRepository) getFullText(text ...string) string { +var quotesRegex = regexp.MustCompile("[“”‘’'\"]") + +func getFullText(text ...string) string { + fullText := sanitizeStrings(text...) + return " " + fullText +} + +func sanitizeStrings(text ...string) string { sanitizedText := strings.Builder{} for _, txt := range text { sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ") @@ -19,14 +27,18 @@ func (r sqlRepository) getFullText(text ...string) string { } var fullText []string for w := range words { - fullText = append(fullText, w) + w = quotesRegex.ReplaceAllString(w, "") + if w != "" { + fullText = append(fullText, w) + } } sort.Strings(fullText) return strings.Join(fullText, " ") } func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, orderBys ...string) error { - q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*")))) + q = strings.TrimSuffix(q, "*") + q = sanitizeStrings(q) if len(q) < 2 { return nil } @@ -37,10 +49,7 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, } parts := strings.Split(q, " ") for _, part := range parts { - sq = sq.Where(Or{ - Like{"full_text": part + "%"}, - Like{"full_text": "%" + part + "%"}, - }) + sq = sq.Where(Like{"full_text": "% " + part + "%"}) } err := r.queryAll(sq, results) return err diff --git a/persistence/sql_search_test.go b/persistence/sql_search_test.go index 64038768f..11a846663 100644 --- a/persistence/sql_search_test.go +++ b/persistence/sql_search_test.go @@ -6,23 +6,25 @@ import ( ) var _ = Describe("sqlRepository", func() { - var sqlRepository = &sqlRepository{} - Describe("getFullText", func() { It("returns all lowercase chars", func() { - Expect(sqlRepository.getFullText("Some Text")).To(Equal("some text")) + Expect(getFullText("Some Text")).To(Equal(" some text")) }) It("removes accents", func() { - Expect(sqlRepository.getFullText("Quintão")).To(Equal("quintao")) + Expect(getFullText("Quintão")).To(Equal(" quintao")) }) It("remove extra spaces", func() { - Expect(sqlRepository.getFullText(" some text ")).To(Equal("some text")) + Expect(getFullText(" some text ")).To(Equal(" some text")) }) It("remove duplicated words", func() { - Expect(sqlRepository.getFullText("legião urbana urbana legiÃo")).To(Equal("legiao urbana")) + Expect(getFullText("legião urbana urbana legiÃo")).To(Equal(" legiao urbana")) + }) + + It("remove symbols", func() { + Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms")) }) }) })