From 877cdf1d5c872ae7fae8a399d3b15f6f6d227fb5 Mon Sep 17 00:00:00 2001 From: Deluan Date: Mon, 8 Feb 2021 10:14:29 -0500 Subject: [PATCH] Get images --- core/agents/cached_http_client.go | 7 ++- core/agents/cached_http_client_test.go | 16 +++++- core/agents/interfaces.go | 4 +- core/agents/lastfm.go | 4 ++ core/agents/spotify.go | 4 ++ core/external_info2.go | 78 +++++++++++++++++++++++--- 6 files changed, 101 insertions(+), 12 deletions(-) diff --git a/core/agents/cached_http_client.go b/core/agents/cached_http_client.go index ba827a275..245bf527d 100644 --- a/core/agents/cached_http_client.go +++ b/core/agents/cached_http_client.go @@ -87,7 +87,12 @@ func (c *CachedHTTPClient) deserializeReq(reqStr string) (*http.Request, error) bodyStr, _ := base64.StdEncoding.DecodeString(*data.Body) body = strings.NewReader(string(bodyStr)) } - return http.NewRequest(data.Method, data.URL, body) + req, err := http.NewRequest(data.Method, data.URL, body) + if err != nil { + return nil, err + } + req.Header = data.Header + return req, nil } func (c *CachedHTTPClient) serializeResponse(resp *http.Response) string { diff --git a/core/agents/cached_http_client_test.go b/core/agents/cached_http_client_test.go index 7d1249b3a..7b484896d 100644 --- a/core/agents/cached_http_client_test.go +++ b/core/agents/cached_http_client_test.go @@ -13,14 +13,16 @@ import ( ) var _ = Describe("CachedHttpClient", func() { - Context("Default TTL", func() { + Context("GET", func() { var chc *CachedHTTPClient var ts *httptest.Server var requestsReceived int + var header string BeforeEach(func() { ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestsReceived++ + header = r.Header.Get("head") _, _ = fmt.Fprintf(w, "Hello, %s", r.URL.Query()["name"]) })) chc = NewCachedHTTPClient(http.DefaultClient, consts.DefaultCachedHttpClientTTL) @@ -56,6 +58,17 @@ var _ = Describe("CachedHttpClient", func() { Expect(err).To(BeNil()) Expect(string(body)).To(Equal("Hello, []")) Expect(requestsReceived).To(Equal(2)) + + // Different again (same as before, but with header) + r, _ = http.NewRequest("GET", ts.URL, nil) + r.Header.Add("head", "this is a header") + resp, err = chc.Do(r) + Expect(err).To(BeNil()) + body, err = ioutil.ReadAll(resp.Body) + Expect(err).To(BeNil()) + Expect(string(body)).To(Equal("Hello, []")) + Expect(string(header)).To(Equal("this is a header")) + Expect(requestsReceived).To(Equal(3)) }) It("expires responses after TTL", func() { @@ -77,5 +90,4 @@ var _ = Describe("CachedHttpClient", func() { Expect(requestsReceived).To(Equal(2)) }) }) - }) diff --git a/core/agents/interfaces.go b/core/agents/interfaces.go index 6deb01093..816daf352 100644 --- a/core/agents/interfaces.go +++ b/core/agents/interfaces.go @@ -7,7 +7,9 @@ import ( type Constructor func(ctx context.Context) Interface -type Interface interface{} +type Interface interface { + AgentName() string +} type Artist struct { Name string diff --git a/core/agents/lastfm.go b/core/agents/lastfm.go index aae714742..da3980fe6 100644 --- a/core/agents/lastfm.go +++ b/core/agents/lastfm.go @@ -32,6 +32,10 @@ func lastFMConstructor(ctx context.Context) Interface { return l } +func (l *lastfmAgent) AgentName() string { + return "lastfm" +} + func (l *lastfmAgent) GetMBID(name string) (string, error) { a, err := l.callArtistGetInfo(name, "") if err != nil { diff --git a/core/agents/spotify.go b/core/agents/spotify.go index 7587cf78f..568165860 100644 --- a/core/agents/spotify.go +++ b/core/agents/spotify.go @@ -37,6 +37,10 @@ func spotifyConstructor(ctx context.Context) Interface { return l } +func (s *spotifyAgent) AgentName() string { + return "spotify" +} + func (s *spotifyAgent) GetImages(name, mbid string) ([]ArtistImage, error) { a, err := s.searchArtist(name) if err != nil { diff --git a/core/external_info2.go b/core/external_info2.go index 0874b0bbf..d83574ba1 100644 --- a/core/external_info2.go +++ b/core/external_info2.go @@ -2,6 +2,7 @@ package core import ( "context" + "sort" "strings" "sync" "time" @@ -65,20 +66,20 @@ func (e *externalInfo2) UpdateArtistInfo(ctx context.Context, id string, similar } // TODO Uncomment - // If we have updated info, just return it + // If we have fresh info, just return it //if time.Since(artist.ExternalInfoUpdatedAt) < consts.ArtistInfoTimeToLive { // log.Debug("Found cached ArtistInfo", "updatedAt", artist.ExternalInfoUpdatedAt, "name", artist.Name) // err := e.loadSimilar(ctx, artist, includeNotPresent) // return artist, err //} - log.Debug("ArtistInfo not cached", "updatedAt", artist.ExternalInfoUpdatedAt, "id", id) + log.Debug(ctx, "ArtistInfo not cached", "updatedAt", artist.ExternalInfoUpdatedAt, "id", id, "name", artist.Name) - wg := sync.WaitGroup{} - e.callGetMBID(ctx, allAgents, artist, &wg) - e.callGetBiography(ctx, allAgents, artist, &wg) - e.callGetURL(ctx, allAgents, artist, &wg) - e.callGetSimilar(ctx, allAgents, artist, similarCount, &wg) - // TODO Images + wg := &sync.WaitGroup{} + e.callGetMBID(ctx, allAgents, artist, wg) + e.callGetBiography(ctx, allAgents, artist, wg) + e.callGetURL(ctx, allAgents, artist, wg) + e.callGetImage(ctx, allAgents, artist, wg) + e.callGetSimilar(ctx, allAgents, artist, similarCount, wg) wg.Wait() artist.ExternalInfoUpdatedAt = time.Now() @@ -111,11 +112,23 @@ func (e *externalInfo2) TopSongs(ctx context.Context, artistName string, count i return nil, nil } +func isDone(ctx context.Context) bool { + select { + case <-ctx.Done(): + return true + default: + return false + } +} + func (e *externalInfo2) callGetMBID(ctx context.Context, allAgents []agents.Interface, artist *model.Artist, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() for _, a := range allAgents { + if isDone(ctx) { + break + } agent, ok := a.(agents.ArtistMBIDRetriever) if !ok { continue @@ -123,6 +136,8 @@ func (e *externalInfo2) callGetMBID(ctx context.Context, allAgents []agents.Inte mbid, err := agent.GetMBID(artist.Name) if mbid != "" && err == nil { artist.MbzArtistID = mbid + log.Debug(ctx, "Got MBID", "agent", a.AgentName(), "artist", artist.Name, "mbid", mbid) + break } } }() @@ -133,6 +148,9 @@ func (e *externalInfo2) callGetURL(ctx context.Context, allAgents []agents.Inter go func() { defer wg.Done() for _, a := range allAgents { + if isDone(ctx) { + break + } agent, ok := a.(agents.ArtistURLRetriever) if !ok { continue @@ -140,6 +158,8 @@ func (e *externalInfo2) callGetURL(ctx context.Context, allAgents []agents.Inter url, err := agent.GetURL(artist.Name, artist.MbzArtistID) if url != "" && err == nil { artist.ExternalUrl = url + log.Debug(ctx, "Got External Url", "agent", a.AgentName(), "artist", artist.Name, "url", url) + break } } }() @@ -150,6 +170,9 @@ func (e *externalInfo2) callGetBiography(ctx context.Context, allAgents []agents go func() { defer wg.Done() for _, a := range allAgents { + if isDone(ctx) { + break + } agent, ok := a.(agents.ArtistBiographyRetriever) if !ok { continue @@ -157,16 +180,53 @@ func (e *externalInfo2) callGetBiography(ctx context.Context, allAgents []agents bio, err := agent.GetBiography(artist.Name, artist.MbzArtistID) if bio != "" && err == nil { artist.Biography = bio + log.Debug(ctx, "Got Biography", "agent", a.AgentName(), "artist", artist.Name, "len", len(bio)) + break } } }() } +func (e *externalInfo2) callGetImage(ctx context.Context, allAgents []agents.Interface, artist *model.Artist, wg *sync.WaitGroup) { + wg.Add(1) + go func() { + defer wg.Done() + for _, a := range allAgents { + if isDone(ctx) { + break + } + agent, ok := a.(agents.ArtistImageRetriever) + if !ok { + continue + } + images, err := agent.GetImages(artist.Name, artist.MbzArtistID) + if len(images) == 0 || err != nil { + continue + } + log.Debug(ctx, "Got Images", "agent", a.AgentName(), "artist", artist.Name, "images", images) + sort.Slice(images, func(i, j int) bool { return images[i].Size > images[j].Size }) + if len(images) >= 1 { + artist.LargeImageUrl = images[0].URL + } + if len(images) >= 2 { + artist.MediumImageUrl = images[1].URL + } + if len(images) >= 3 { + artist.SmallImageUrl = images[2].URL + } + break + } + }() +} + func (e *externalInfo2) callGetSimilar(ctx context.Context, allAgents []agents.Interface, artist *model.Artist, limit int, wg *sync.WaitGroup) { wg.Add(1) go func() { defer wg.Done() for _, a := range allAgents { + if isDone(ctx) { + break + } agent, ok := a.(agents.ArtistSimilarRetriever) if !ok { continue @@ -179,7 +239,9 @@ func (e *externalInfo2) callGetSimilar(ctx context.Context, allAgents []agents.I if err != nil { continue } + log.Debug(ctx, "Got Similar Artists", "agent", a.AgentName(), "artist", artist.Name, "similar", similar) artist.SimilarArtists = sa + break } }() }