From 6a3fd46219ff36e77ae2f5793c5042baacf556e9 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 31 Oct 2023 19:26:41 +0100 Subject: [PATCH 1/7] Fix GH-264 --- dev/releases/3.2.yml | 8 +++++++- maloja/malojatime.py | 2 -- maloja/pkg_global/conf.py | 4 ++-- maloja/thirdparty/spotify.py | 4 +++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/dev/releases/3.2.yml b/dev/releases/3.2.yml index bcd0d53..03880a2 100644 --- a/dev/releases/3.2.yml +++ b/dev/releases/3.2.yml @@ -21,4 +21,10 @@ minor_release_name: "Nicole" - "[Technical] Bumped Python and SQLAlchemy versions" - "[Distribution] Removed build of arm/v7 image" 3.2.1: - notes: [] \ No newline at end of file + notes: + - "[Feature] Added setting for custom week offset" + - "[Bugfix] Fixed album entity rows being marked as track entity rows" + - "[Bugfix] Fixed scrobbling of tracks when all artists have been removed by server parsing" + - "[Bugfix] Fixed Spotify import of multiple files" + - "[Bugfix] Fixed process control on FreeBSD" + - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating" \ No newline at end of file diff --git a/maloja/malojatime.py b/maloja/malojatime.py index 9561fac..1af046d 100644 --- a/maloja/malojatime.py +++ b/maloja/malojatime.py @@ -214,8 +214,6 @@ class MTRangeWeek(MTRangeSingular): # do this so we can construct the week with overflow (eg 2020/-3) thisisoyear_firstday = date.fromisocalendar(year,1,1) + timedelta(days=malojaconfig['WEEK_OFFSET']-1) self.firstday = thisisoyear_firstday + timedelta(days=7*(week-1)) - self.firstday = date(self.firstday.year,self.firstday.month,self.firstday.day) - # for compatibility with pre python3.8 (https://bugs.python.org/issue32417) self.lastday = self.firstday + timedelta(days=6) diff --git a/maloja/pkg_global/conf.py b/maloja/pkg_global/conf.py index 36a6566..2ff5026 100644 --- a/maloja/pkg_global/conf.py +++ b/maloja/pkg_global/conf.py @@ -190,7 +190,8 @@ malojaconfig = Configuration( "delimiters_formal":(tp.Set(tp.String()), "Formal Delimiters", [";","/","|","␝","␞","␟"], "Delimiters used to tag multiple artists when only one tag field is available"), "filters_remix":(tp.Set(tp.String()), "Remix Filters", ["Remix", "Remix Edit", "Short Mix", "Extended Mix", "Soundtrack Version"], "Filters used to recognize the remix artists in the title"), "parse_remix_artists":(tp.Boolean(), "Parse Remix Artists", False), - "week_offset":(tp.Integer(), "Week Begin Offset", 0, "Start of the week for the purpose of weekly statistics. 0 = Sunday, 6 = Saturday") + "week_offset":(tp.Integer(), "Week Begin Offset", 0, "Start of the week for the purpose of weekly statistics. 0 = Sunday, 6 = Saturday"), + "timezone":(tp.Integer(), "UTC Offset", 0) }, "Web Interface":{ "default_range_startpage":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}), "Default Range for Startpage Stats", "year"), @@ -204,7 +205,6 @@ malojaconfig = Configuration( "discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"), "use_local_images":(tp.Boolean(), "Use Local Images", True), #"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600), - "timezone":(tp.Integer(), "UTC Offset", 0), "time_format":(tp.String(), "Time Format", "%d. %b %Y %I:%M %p"), "theme":(tp.String(), "Theme", "maloja") } diff --git a/maloja/thirdparty/spotify.py b/maloja/thirdparty/spotify.py index 3a8e55f..9d9e4b9 100644 --- a/maloja/thirdparty/spotify.py +++ b/maloja/thirdparty/spotify.py @@ -47,6 +47,8 @@ class Spotify(MetadataInterface): expire = responsedata.get("expires_in",3600) self.settings["token"] = responsedata["access_token"] #log("Successfully authenticated with Spotify") - Timer(expire,self.authorize).start() + t = Timer(expire,self.authorize) + t.daemon = True + t.start() except Exception as e: log("Error while authenticating with Spotify: " + repr(e)) From 7910ee3b6b5e89da8f77addd6bf970d291550140 Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 1 Nov 2023 16:37:27 +0100 Subject: [PATCH 2/7] Improve track image fetching from Musicbrainz --- maloja/__main__.py | 11 ++++++++- maloja/thirdparty/musicbrainz.py | 39 +++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/maloja/__main__.py b/maloja/__main__.py index 12a77c5..0fa7e7d 100644 --- a/maloja/__main__.py +++ b/maloja/__main__.py @@ -160,6 +160,14 @@ def print_info(): except Exception: print("Could not determine system information.") + +def print_settings(): + print_header_info() + maxlen = max(len(k) for k in conf.malojaconfig) + for k in conf.malojaconfig: + print(col['lightblue'](k.ljust(maxlen+2)),conf.malojaconfig[k]) + + @mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images','prefer_existing'],shield=True) def main(*args,**kwargs): @@ -180,7 +188,8 @@ def main(*args,**kwargs): "apidebug":apidebug.run, # maloja apidebug "parsealbums":tasks.parse_albums, # maloja parsealbums --strategy majority # aux - "info":print_info + "info":print_info, + "settings":print_settings } if "version" in kwargs: diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index f16229b..56e0be1 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -33,14 +33,15 @@ class MusicBrainz(MetadataInterface): self.lock.acquire() try: artists, title = track - artiststring = ", ".join(artists) #Join artists collection into string - titlestring = title + searchstr = f'recording:"{title}"' + for artist in artists: + searchstr += f' artist:"{artist}"' querystr = urllib.parse.urlencode({ "fmt":"json", - "query":"{title} {artist}".format(artist=artiststring,title=titlestring) + "query":searchstr }) req = urllib.request.Request(**{ - "url":"https://musicbrainz.org/ws/2/release?" + querystr, + "url":"https://musicbrainz.org/ws/2/recording?" + querystr, "method":"GET", "headers":{ "User-Agent":self.useragent @@ -49,15 +50,27 @@ class MusicBrainz(MetadataInterface): response = urllib.request.urlopen(req) responsedata = response.read() data = json.loads(responsedata) - mbid = data["releases"][0]["id"] - response = urllib.request.urlopen( - "https://coverartarchive.org/release/{mbid}?fmt=json".format(mbid=mbid) - ) - responsedata = response.read() - data = json.loads(responsedata) - imgurl = self.metadata_parse_response_track(data) - if imgurl is not None: imgurl = self.postprocess_url(imgurl) - return imgurl + entity = data["recordings"][0]["releases"][0] + coverartendpoint = "release" + while True: + mbid = entity["id"] + try: + response = urllib.request.urlopen( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json" + ) + responsedata = response.read() + data = json.loads(responsedata) + imgurl = self.metadata_parse_response_track(data) + except: + imgurl = None + if imgurl is None: + entity = entity["release-group"] + # this will raise an error so we don't stay in the while loop forever + coverartendpoint = "release-group" + continue + + imgurl = self.postprocess_url(imgurl) + return imgurl except Exception: return None From 139de02d9a0ff672c28574e6f553ebbb9746fb96 Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 1 Nov 2023 17:01:03 +0100 Subject: [PATCH 3/7] Add album art fetching from Musicbrainz, fix GH-265 --- dev/releases/3.2.yml | 1 + maloja/pkg_global/conf.py | 2 +- maloja/thirdparty/musicbrainz.py | 62 ++++++++++++++++++++++++++++++-- 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/dev/releases/3.2.yml b/dev/releases/3.2.yml index 03880a2..ca388eb 100644 --- a/dev/releases/3.2.yml +++ b/dev/releases/3.2.yml @@ -23,6 +23,7 @@ minor_release_name: "Nicole" 3.2.1: notes: - "[Feature] Added setting for custom week offset" + - "[Feature] Added Musicbrainz album art fetching" - "[Bugfix] Fixed album entity rows being marked as track entity rows" - "[Bugfix] Fixed scrobbling of tracks when all artists have been removed by server parsing" - "[Bugfix] Fixed Spotify import of multiple files" diff --git a/maloja/pkg_global/conf.py b/maloja/pkg_global/conf.py index 2ff5026..5ed6052 100644 --- a/maloja/pkg_global/conf.py +++ b/maloja/pkg_global/conf.py @@ -201,7 +201,7 @@ malojaconfig = Configuration( "display_art_icons":(tp.Boolean(), "Display Album/Artist Icons", True), "default_album_artist":(tp.String(), "Default Albumartist", "Various Artists"), "use_album_artwork_for_tracks":(tp.Boolean(), "Use Album Artwork for tracks", True), - "fancy_placeholder_art":(tp.Boolean(), "Use fancy placeholder artwork",True), + "fancy_placeholder_art":(tp.Boolean(), "Use fancy placeholder artwork",False), "discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"), "use_local_images":(tp.Boolean(), "Use Local Images", True), #"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600), diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index 56e0be1..6f1f2b1 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -13,12 +13,14 @@ class MusicBrainz(MetadataInterface): lock = threading.Lock() useragent = USER_AGENT + + thumbnailsize_order = ['500','large','1200','250','small'] + settings = { } metadata = { "response_type":"json", - "response_parse_tree_track": ["images",0,"thumbnails","500"], "required_settings": [], } @@ -27,7 +29,57 @@ class MusicBrainz(MetadataInterface): # not supported def get_image_album(self,album): - return None + self.lock.acquire() + try: + artists, title = album + searchstr = f'release:"{title}"' + for artist in artists: + searchstr += f' artist:"{artist}"' + querystr = urllib.parse.urlencode({ + "fmt":"json", + "query":searchstr + }) + req = urllib.request.Request(**{ + "url":"https://musicbrainz.org/ws/2/release?" + querystr, + "method":"GET", + "headers":{ + "User-Agent":self.useragent + } + }) + response = urllib.request.urlopen(req) + responsedata = response.read() + data = json.loads(responsedata) + entity = data["releases"][0] + coverartendpoint = "release" + while True: + mbid = entity["id"] + try: + response = urllib.request.urlopen( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json" + ) + responsedata = response.read() + data = json.loads(responsedata) + thumbnails = data['images'][0]['thumbnails'] + for size in self.thumbnailsize_order: + if thumbnails.get(size) is not None: + imgurl = thumbnails.get(size) + continue + except: + imgurl = None + if imgurl is None: + entity = entity["release-group"] + # this will raise an error so we don't stay in the while loop forever + coverartendpoint = "release-group" + continue + + imgurl = self.postprocess_url(imgurl) + return imgurl + + except Exception: + return None + finally: + time.sleep(2) + self.lock.release() def get_image_track(self,track): self.lock.acquire() @@ -60,7 +112,11 @@ class MusicBrainz(MetadataInterface): ) responsedata = response.read() data = json.loads(responsedata) - imgurl = self.metadata_parse_response_track(data) + thumbnails = data['images'][0]['thumbnails'] + for size in self.thumbnailsize_order: + if thumbnails.get(size) is not None: + imgurl = thumbnails.get(size) + continue except: imgurl = None if imgurl is None: From 76691a5b0fcd6030b34bddd6b2ac855b1ac6043a Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 1 Nov 2023 19:01:29 +0100 Subject: [PATCH 4/7] Upgrade third party modules to use requests --- dev/releases/3.2.yml | 3 +- maloja/thirdparty/__init__.py | 58 ++++++++++++++++-------------- maloja/thirdparty/lastfm.py | 31 ++++++++++------ maloja/thirdparty/maloja.py | 6 ++-- maloja/thirdparty/musicbrainz.py | 62 ++++++++++++++++---------------- maloja/thirdparty/spotify.py | 19 +++++----- 6 files changed, 98 insertions(+), 81 deletions(-) diff --git a/dev/releases/3.2.yml b/dev/releases/3.2.yml index ca388eb..cf75ee3 100644 --- a/dev/releases/3.2.yml +++ b/dev/releases/3.2.yml @@ -28,4 +28,5 @@ minor_release_name: "Nicole" - "[Bugfix] Fixed scrobbling of tracks when all artists have been removed by server parsing" - "[Bugfix] Fixed Spotify import of multiple files" - "[Bugfix] Fixed process control on FreeBSD" - - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating" \ No newline at end of file + - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating" + - "[Technical] Upgraded all third party modules to use requests module and send User Agent" \ No newline at end of file diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index 7c582d0..7b45e38 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -8,7 +8,8 @@ import xml.etree.ElementTree as ElementTree import json -import urllib.parse, urllib.request +import requests +import urllib.parse import base64 import time from doreah.logging import log @@ -16,6 +17,7 @@ from threading import BoundedSemaphore from ..pkg_global.conf import malojaconfig from .. import database +from ..__pkginfo__ import USER_AGENT services = { @@ -100,6 +102,8 @@ class GenericInterface: scrobbleimport = {} metadata = {} + useragent = USER_AGENT + def __init__(self): # populate from settings file once on creation # avoid constant disk access, restart on adding services is acceptable @@ -127,16 +131,6 @@ class GenericInterface: return True # per default. no authorization is necessary - # wrapper method - def request(self,url,data,responsetype): - response = urllib.request.urlopen( - url, - data=utf(data) - ) - responsedata = response.read() - if responsetype == "xml": - data = ElementTree.fromstring(responsedata) - return data # proxy scrobbler class ProxyScrobbleInterface(GenericInterface,abstract=True): @@ -155,11 +149,15 @@ class ProxyScrobbleInterface(GenericInterface,abstract=True): ) def scrobble(self,artists,title,timestamp): - response = urllib.request.urlopen( - self.proxyscrobble["scrobbleurl"], - data=utf(self.proxyscrobble_postdata(artists,title,timestamp))) - responsedata = response.read() + response = requests.post( + url=self.proxyscrobble["scrobbleurl"], + data=self.proxyscrobble_postdata(artists,title,timestamp), + headers={ + "User-Agent":self.useragent + } + ) if self.proxyscrobble["response_type"] == "xml": + responsedata = response.text data = ElementTree.fromstring(responsedata) return self.proxyscrobble_parse_response(data) @@ -211,13 +209,15 @@ class MetadataInterface(GenericInterface,abstract=True): artists, title = track artiststring = urllib.parse.quote(", ".join(artists)) titlestring = urllib.parse.quote(title) - response = urllib.request.urlopen( - self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings) + response = requests.get( + self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_track(data) else: imgurl = None @@ -227,13 +227,15 @@ class MetadataInterface(GenericInterface,abstract=True): def get_image_artist(self,artist): artiststring = urllib.parse.quote(artist) - response = urllib.request.urlopen( - self.metadata["artisturl"].format(artist=artiststring,**self.settings) + response = requests.get( + self.metadata["artisturl"].format(artist=artiststring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_artist(data) else: imgurl = None @@ -245,13 +247,15 @@ class MetadataInterface(GenericInterface,abstract=True): artists, title = album artiststring = urllib.parse.quote(", ".join(artists or [])) titlestring = urllib.parse.quote(title) - response = urllib.request.urlopen( - self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings) + response = requests.get( + self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_album(data) else: imgurl = None diff --git a/maloja/thirdparty/lastfm.py b/maloja/thirdparty/lastfm.py index e565e5f..ff2ec65 100644 --- a/maloja/thirdparty/lastfm.py +++ b/maloja/thirdparty/lastfm.py @@ -1,6 +1,7 @@ from . import MetadataInterface, ProxyScrobbleInterface, utf import hashlib -import urllib.parse, urllib.request +import requests +import xml.etree.ElementTree as ElementTree from doreah.logging import log class LastFM(MetadataInterface, ProxyScrobbleInterface): @@ -54,27 +55,37 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface): def authorize(self): try: - result = self.request( - self.proxyscrobble['scrobbleurl'], - self.query_compose({ + response = requests.post( + url=self.proxyscrobble['scrobbleurl'], + params=self.query_compose({ "method":"auth.getMobileSession", "username":self.settings["username"], "password":self.settings["password"], "api_key":self.settings["apikey"] }), - responsetype="xml" + headers={ + "User-Agent":self.useragent + } ) - self.settings["sk"] = result.find("session").findtext("key") + + data = ElementTree.fromstring(response.text) + self.settings["sk"] = data.find("session").findtext("key") except Exception as e: - pass - #log("Error while authenticating with LastFM: " + repr(e)) + log("Error while authenticating with LastFM: " + repr(e)) - # creates signature and returns full query string + # creates signature and returns full query def query_compose(self,parameters): m = hashlib.md5() keys = sorted(str(k) for k in parameters) m.update(utf("".join(str(k) + str(parameters[k]) for k in keys))) m.update(utf(self.settings["secret"])) sig = m.hexdigest() - return urllib.parse.urlencode(parameters) + "&api_sig=" + sig + return {**parameters,"api_sig":sig} + + def handle_json_result_error(self,result): + if "track" in result and not result.get("track").get('album',{}): + return True + + if "error" in result and result.get("error") == 6: + return True diff --git a/maloja/thirdparty/maloja.py b/maloja/thirdparty/maloja.py index 53413f8..4975400 100644 --- a/maloja/thirdparty/maloja.py +++ b/maloja/thirdparty/maloja.py @@ -1,5 +1,5 @@ from . import ProxyScrobbleInterface, ImportInterface -import urllib.request +import requests from doreah.logging import log import json @@ -32,8 +32,8 @@ class OtherMalojaInstance(ProxyScrobbleInterface, ImportInterface): def get_remote_scrobbles(self): url = f"{self.settings['instance']}/apis/mlj_1/scrobbles" - response = urllib.request.urlopen(url) - data = json.loads(response.read().decode('utf-8')) + response = requests.get(url) + data = response.json() for scrobble in data['list']: yield scrobble diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index 6f1f2b1..5dc6f72 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -1,9 +1,7 @@ from . import MetadataInterface -import urllib.parse, urllib.request -import json +import requests import time import threading -from ..__pkginfo__ import USER_AGENT class MusicBrainz(MetadataInterface): name = "MusicBrainz" @@ -11,7 +9,6 @@ class MusicBrainz(MetadataInterface): # musicbrainz is rate-limited lock = threading.Lock() - useragent = USER_AGENT thumbnailsize_order = ['500','large','1200','250','small'] @@ -35,30 +32,32 @@ class MusicBrainz(MetadataInterface): searchstr = f'release:"{title}"' for artist in artists: searchstr += f' artist:"{artist}"' - querystr = urllib.parse.urlencode({ - "fmt":"json", - "query":searchstr - }) - req = urllib.request.Request(**{ - "url":"https://musicbrainz.org/ws/2/release?" + querystr, - "method":"GET", + res = requests.get(**{ + "url":"https://musicbrainz.org/ws/2/release", + "params":{ + "fmt":"json", + "query":searchstr + }, "headers":{ "User-Agent":self.useragent } }) - response = urllib.request.urlopen(req) - responsedata = response.read() - data = json.loads(responsedata) + data = res.json() entity = data["releases"][0] coverartendpoint = "release" while True: mbid = entity["id"] try: - response = urllib.request.urlopen( - f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json" + response = requests.get( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}", + params={ + "fmt":"json" + }, + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() - data = json.loads(responsedata) + data = response.json() thumbnails = data['images'][0]['thumbnails'] for size in self.thumbnailsize_order: if thumbnails.get(size) is not None: @@ -88,30 +87,29 @@ class MusicBrainz(MetadataInterface): searchstr = f'recording:"{title}"' for artist in artists: searchstr += f' artist:"{artist}"' - querystr = urllib.parse.urlencode({ - "fmt":"json", - "query":searchstr - }) - req = urllib.request.Request(**{ - "url":"https://musicbrainz.org/ws/2/recording?" + querystr, - "method":"GET", + res = requests.get(**{ + "url":"https://musicbrainz.org/ws/2/recording", + "params":{ + "fmt":"json", + "query":searchstr + }, "headers":{ "User-Agent":self.useragent } }) - response = urllib.request.urlopen(req) - responsedata = response.read() - data = json.loads(responsedata) + data = res.json() entity = data["recordings"][0]["releases"][0] coverartendpoint = "release" while True: mbid = entity["id"] try: - response = urllib.request.urlopen( - f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json" + response = requests.get( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}", + params={ + "fmt":"json" + } ) - responsedata = response.read() - data = json.loads(responsedata) + data = response.json() thumbnails = data['images'][0]['thumbnails'] for size in self.thumbnailsize_order: if thumbnails.get(size) is not None: diff --git a/maloja/thirdparty/spotify.py b/maloja/thirdparty/spotify.py index 9d9e4b9..03d614d 100644 --- a/maloja/thirdparty/spotify.py +++ b/maloja/thirdparty/spotify.py @@ -1,6 +1,5 @@ from . import MetadataInterface, utf, b64 -import urllib.parse, urllib.request -import json +import requests from threading import Timer from doreah.logging import log @@ -31,15 +30,14 @@ class Spotify(MetadataInterface): try: keys = { "url":"https://accounts.spotify.com/api/token", - "method":"POST", "headers":{ - "Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8") + "Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8"), + "User-Agent": self.useragent }, - "data":bytes(urllib.parse.urlencode({"grant_type":"client_credentials"}),encoding="utf-8") + "data":{"grant_type":"client_credentials"} } - req = urllib.request.Request(**keys) - response = urllib.request.urlopen(req) - responsedata = json.loads(response.read()) + res = requests.post(**keys) + responsedata = res.json() if "error" in responsedata: log("Error authenticating with Spotify: " + responsedata['error_description']) expire = 3600 @@ -52,3 +50,8 @@ class Spotify(MetadataInterface): t.start() except Exception as e: log("Error while authenticating with Spotify: " + repr(e)) + + def handle_json_result_error(self,result): + result = result.get('tracks') or result.get('albums') or result.get('artists') + if not result['items']: + return True \ No newline at end of file From d0a20fecb2c9f0adec7ef0e50a9867ccbf9f07e7 Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 1 Nov 2023 20:07:05 +0100 Subject: [PATCH 5/7] Fix misleading logging entries about metadata requests --- maloja/pkg_global/conf.py | 2 +- maloja/thirdparty/__init__.py | 11 ++++++++--- maloja/thirdparty/audiodb.py | 1 + maloja/thirdparty/deezer.py | 1 + maloja/thirdparty/lastfm.py | 1 + maloja/thirdparty/musicbrainz.py | 1 + maloja/thirdparty/spotify.py | 1 + 7 files changed, 14 insertions(+), 4 deletions(-) diff --git a/maloja/pkg_global/conf.py b/maloja/pkg_global/conf.py index 5ed6052..89a5cf3 100644 --- a/maloja/pkg_global/conf.py +++ b/maloja/pkg_global/conf.py @@ -164,7 +164,7 @@ malojaconfig = Configuration( "name":(tp.String(), "Name", "Generic Maloja User") }, "Third Party Services":{ - "metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','musicbrainz'], "Which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."), + "metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "Which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."), "scrobble_lastfm":(tp.Boolean(), "Proxy-Scrobble to Last.fm", False), "lastfm_api_key":(tp.String(), "Last.fm API Key", None), "lastfm_api_secret":(tp.String(), "Last.fm API Secret", None), diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index 7b45e38..baffd83 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -7,13 +7,12 @@ # pls don't sue me import xml.etree.ElementTree as ElementTree -import json import requests import urllib.parse import base64 import time from doreah.logging import log -from threading import BoundedSemaphore +from threading import BoundedSemaphore, Thread from ..pkg_global.conf import malojaconfig from .. import database @@ -53,6 +52,7 @@ def proxy_scrobble_all(artists,title,timestamp): def get_image_track_all(track): with thirdpartylock: for service in services["metadata"]: + if "track" not in service.metadata["enabled_entity_types"]: continue try: res = service.get_image_track(track) if res: @@ -65,6 +65,7 @@ def get_image_track_all(track): def get_image_artist_all(artist): with thirdpartylock: for service in services["metadata"]: + if "artist" not in service.metadata["enabled_entity_types"]: continue try: res = service.get_image_artist(artist) if res: @@ -77,6 +78,7 @@ def get_image_artist_all(artist): def get_image_album_all(album): with thirdpartylock: for service in services["metadata"]: + if "album" not in service.metadata["enabled_entity_types"]: continue try: res = service.get_image_album(album) if res: @@ -109,7 +111,10 @@ class GenericInterface: # avoid constant disk access, restart on adding services is acceptable for key in self.settings: self.settings[key] = malojaconfig[self.settings[key]] - self.authorize() + t = Thread(target=self.authorize) + t.daemon = True + t.start() + #self.authorize() # this makes sure that of every class we define, we immediately create an # instance (de facto singleton). then each instance checks if the requirements diff --git a/maloja/thirdparty/audiodb.py b/maloja/thirdparty/audiodb.py index 9a4d84a..5b9e71f 100644 --- a/maloja/thirdparty/audiodb.py +++ b/maloja/thirdparty/audiodb.py @@ -16,6 +16,7 @@ class AudioDB(MetadataInterface): #"response_parse_tree_track": ["tracks",0,"astrArtistThumb"], "response_parse_tree_artist": ["artists",0,"strArtistThumb"], "required_settings": ["api_key"], + "enabled_entity_types": ["artist"] } def get_image_track(self,track): diff --git a/maloja/thirdparty/deezer.py b/maloja/thirdparty/deezer.py index dfac7a8..f30a9ba 100644 --- a/maloja/thirdparty/deezer.py +++ b/maloja/thirdparty/deezer.py @@ -17,6 +17,7 @@ class Deezer(MetadataInterface): "response_parse_tree_artist": ["data",0,"artist","picture_medium"], "response_parse_tree_album": ["data",0,"album","cover_medium"], "required_settings": [], + "enabled_entity_types": ["artist","album"] } delay = 1 diff --git a/maloja/thirdparty/lastfm.py b/maloja/thirdparty/lastfm.py index ff2ec65..cfd79b0 100644 --- a/maloja/thirdparty/lastfm.py +++ b/maloja/thirdparty/lastfm.py @@ -32,6 +32,7 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface): #"response_parse_tree_artist": ["artist","image",-1,"#text"], "response_parse_tree_album": ["album","image",-1,"#text"], "required_settings": ["apikey"], + "enabled_entity_types": ["track","album"] } def get_image_artist(self,artist): diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index 5dc6f72..668f7f3 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -19,6 +19,7 @@ class MusicBrainz(MetadataInterface): metadata = { "response_type":"json", "required_settings": [], + "enabled_entity_types": ["album","track"] } def get_image_artist(self,artist): diff --git a/maloja/thirdparty/spotify.py b/maloja/thirdparty/spotify.py index 03d614d..93f4d62 100644 --- a/maloja/thirdparty/spotify.py +++ b/maloja/thirdparty/spotify.py @@ -21,6 +21,7 @@ class Spotify(MetadataInterface): "response_parse_tree_album": ["albums","items",0,"images",0,"url"], "response_parse_tree_artist": ["artists","items",0,"images",0,"url"], "required_settings": ["apiid","secret"], + "enabled_entity_types": ["artist","album","track"] } def authorize(self): From 5495d6e38d95c0c2128e1de9a9553b55b6be945b Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 2 Nov 2023 00:36:57 +0100 Subject: [PATCH 6/7] Version bump --- maloja/__pkginfo__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 873e9e7..47c8e25 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "3.2.0" +VERSION = "3.2.1" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/pyproject.toml b/pyproject.toml index 275e135..e543872 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "3.2.0" +version = "3.2.1" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.10" From 6bb81ce589ea3e0f25167687d101793bbafcb122 Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 3 Nov 2023 13:41:30 +0100 Subject: [PATCH 7/7] Fix GH-271 --- dev/releases/3.2.yml | 6 +++++- maloja/thirdparty/lastfm.py | 35 ++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/dev/releases/3.2.yml b/dev/releases/3.2.yml index cf75ee3..9d96402 100644 --- a/dev/releases/3.2.yml +++ b/dev/releases/3.2.yml @@ -21,6 +21,7 @@ minor_release_name: "Nicole" - "[Technical] Bumped Python and SQLAlchemy versions" - "[Distribution] Removed build of arm/v7 image" 3.2.1: + commit: "5495d6e38d95c0c2128e1de9a9553b55b6be945b" notes: - "[Feature] Added setting for custom week offset" - "[Feature] Added Musicbrainz album art fetching" @@ -29,4 +30,7 @@ minor_release_name: "Nicole" - "[Bugfix] Fixed Spotify import of multiple files" - "[Bugfix] Fixed process control on FreeBSD" - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating" - - "[Technical] Upgraded all third party modules to use requests module and send User Agent" \ No newline at end of file + - "[Technical] Upgraded all third party modules to use requests module and send User Agent" +3.2.2: + notes: + - "[Bugfix] Fixed Last.fm authentication" \ No newline at end of file diff --git a/maloja/thirdparty/lastfm.py b/maloja/thirdparty/lastfm.py index cfd79b0..897fef3 100644 --- a/maloja/thirdparty/lastfm.py +++ b/maloja/thirdparty/lastfm.py @@ -55,24 +55,25 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface): }) def authorize(self): - try: - response = requests.post( - url=self.proxyscrobble['scrobbleurl'], - params=self.query_compose({ - "method":"auth.getMobileSession", - "username":self.settings["username"], - "password":self.settings["password"], - "api_key":self.settings["apikey"] - }), - headers={ - "User-Agent":self.useragent - } - ) + if all(self.settings[key] not in [None,"ASK",False] for key in ["username","password","apikey","secret"]): + try: + response = requests.post( + url=self.proxyscrobble['scrobbleurl'], + params=self.query_compose({ + "method":"auth.getMobileSession", + "username":self.settings["username"], + "password":self.settings["password"], + "api_key":self.settings["apikey"] + }), + headers={ + "User-Agent":self.useragent + } + ) - data = ElementTree.fromstring(response.text) - self.settings["sk"] = data.find("session").findtext("key") - except Exception as e: - log("Error while authenticating with LastFM: " + repr(e)) + data = ElementTree.fromstring(response.text) + self.settings["sk"] = data.find("session").findtext("key") + except Exception as e: + log("Error while authenticating with LastFM: " + repr(e)) # creates signature and returns full query