diff --git a/dev/releases/3.2.yml b/dev/releases/3.2.yml index bcd0d53..9d96402 100644 --- a/dev/releases/3.2.yml +++ b/dev/releases/3.2.yml @@ -21,4 +21,16 @@ minor_release_name: "Nicole" - "[Technical] Bumped Python and SQLAlchemy versions" - "[Distribution] Removed build of arm/v7 image" 3.2.1: - notes: [] \ No newline at end of file + commit: "5495d6e38d95c0c2128e1de9a9553b55b6be945b" + notes: + - "[Feature] Added setting for custom week offset" + - "[Feature] Added Musicbrainz album art fetching" + - "[Bugfix] Fixed album entity rows being marked as track entity rows" + - "[Bugfix] Fixed scrobbling of tracks when all artists have been removed by server parsing" + - "[Bugfix] Fixed Spotify import of multiple files" + - "[Bugfix] Fixed process control on FreeBSD" + - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating" + - "[Technical] Upgraded all third party modules to use requests module and send User Agent" +3.2.2: + notes: + - "[Bugfix] Fixed Last.fm authentication" \ No newline at end of file diff --git a/maloja/__main__.py b/maloja/__main__.py index 12a77c5..0fa7e7d 100644 --- a/maloja/__main__.py +++ b/maloja/__main__.py @@ -160,6 +160,14 @@ def print_info(): except Exception: print("Could not determine system information.") + +def print_settings(): + print_header_info() + maxlen = max(len(k) for k in conf.malojaconfig) + for k in conf.malojaconfig: + print(col['lightblue'](k.ljust(maxlen+2)),conf.malojaconfig[k]) + + @mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images','prefer_existing'],shield=True) def main(*args,**kwargs): @@ -180,7 +188,8 @@ def main(*args,**kwargs): "apidebug":apidebug.run, # maloja apidebug "parsealbums":tasks.parse_albums, # maloja parsealbums --strategy majority # aux - "info":print_info + "info":print_info, + "settings":print_settings } if "version" in kwargs: diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 873e9e7..47c8e25 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "3.2.0" +VERSION = "3.2.1" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/maloja/malojatime.py b/maloja/malojatime.py index 9561fac..1af046d 100644 --- a/maloja/malojatime.py +++ b/maloja/malojatime.py @@ -214,8 +214,6 @@ class MTRangeWeek(MTRangeSingular): # do this so we can construct the week with overflow (eg 2020/-3) thisisoyear_firstday = date.fromisocalendar(year,1,1) + timedelta(days=malojaconfig['WEEK_OFFSET']-1) self.firstday = thisisoyear_firstday + timedelta(days=7*(week-1)) - self.firstday = date(self.firstday.year,self.firstday.month,self.firstday.day) - # for compatibility with pre python3.8 (https://bugs.python.org/issue32417) self.lastday = self.firstday + timedelta(days=6) diff --git a/maloja/pkg_global/conf.py b/maloja/pkg_global/conf.py index a560d5d..98013d1 100644 --- a/maloja/pkg_global/conf.py +++ b/maloja/pkg_global/conf.py @@ -164,7 +164,7 @@ malojaconfig = Configuration( "name":(tp.String(), "Name", "Generic Maloja User") }, "Third Party Services":{ - "metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','musicbrainz'], "Which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."), + "metadata_providers":(tp.List(tp.String()), "Metadata Providers", ['lastfm','spotify','deezer','audiodb','musicbrainz'], "Which metadata providers should be used in what order. Musicbrainz is rate-limited and should not be used first."), "scrobble_lastfm":(tp.Boolean(), "Proxy-Scrobble to Last.fm", False), "lastfm_api_key":(tp.String(), "Last.fm API Key", None), "lastfm_api_secret":(tp.String(), "Last.fm API Secret", None), @@ -190,7 +190,8 @@ malojaconfig = Configuration( "delimiters_formal":(tp.Set(tp.String()), "Formal Delimiters", [";","/","|","␝","␞","␟"], "Delimiters used to tag multiple artists when only one tag field is available"), "filters_remix":(tp.Set(tp.String()), "Remix Filters", ["Remix", "Remix Edit", "Short Mix", "Extended Mix", "Soundtrack Version"], "Filters used to recognize the remix artists in the title"), "parse_remix_artists":(tp.Boolean(), "Parse Remix Artists", False), - "week_offset":(tp.Integer(), "Week Begin Offset", 0, "Start of the week for the purpose of weekly statistics. 0 = Sunday, 6 = Saturday") + "week_offset":(tp.Integer(), "Week Begin Offset", 0, "Start of the week for the purpose of weekly statistics. 0 = Sunday, 6 = Saturday"), + "timezone":(tp.Integer(), "UTC Offset", 0) }, "Web Interface":{ "default_range_startpage":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}), "Default Range for Startpage Stats", "year"), @@ -200,12 +201,11 @@ malojaconfig = Configuration( "display_art_icons":(tp.Boolean(), "Display Album/Artist Icons", True), "default_album_artist":(tp.String(), "Default Albumartist", "Various Artists"), "use_album_artwork_for_tracks":(tp.Boolean(), "Use Album Artwork for tracks", True), - "fancy_placeholder_art":(tp.Boolean(), "Use fancy placeholder artwork",True), + "fancy_placeholder_art":(tp.Boolean(), "Use fancy placeholder artwork",False), "show_play_number_on_tiles":(tp.Boolean(), "Show amount of plays on tails", True), "discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"), "use_local_images":(tp.Boolean(), "Use Local Images", True), #"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600), - "timezone":(tp.Integer(), "UTC Offset", 0), "time_format":(tp.String(), "Time Format", "%d. %b %Y %I:%M %p"), "theme":(tp.String(), "Theme", "maloja") } diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index 7c582d0..baffd83 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -7,15 +7,16 @@ # pls don't sue me import xml.etree.ElementTree as ElementTree -import json -import urllib.parse, urllib.request +import requests +import urllib.parse import base64 import time from doreah.logging import log -from threading import BoundedSemaphore +from threading import BoundedSemaphore, Thread from ..pkg_global.conf import malojaconfig from .. import database +from ..__pkginfo__ import USER_AGENT services = { @@ -51,6 +52,7 @@ def proxy_scrobble_all(artists,title,timestamp): def get_image_track_all(track): with thirdpartylock: for service in services["metadata"]: + if "track" not in service.metadata["enabled_entity_types"]: continue try: res = service.get_image_track(track) if res: @@ -63,6 +65,7 @@ def get_image_track_all(track): def get_image_artist_all(artist): with thirdpartylock: for service in services["metadata"]: + if "artist" not in service.metadata["enabled_entity_types"]: continue try: res = service.get_image_artist(artist) if res: @@ -75,6 +78,7 @@ def get_image_artist_all(artist): def get_image_album_all(album): with thirdpartylock: for service in services["metadata"]: + if "album" not in service.metadata["enabled_entity_types"]: continue try: res = service.get_image_album(album) if res: @@ -100,12 +104,17 @@ class GenericInterface: scrobbleimport = {} metadata = {} + useragent = USER_AGENT + def __init__(self): # populate from settings file once on creation # avoid constant disk access, restart on adding services is acceptable for key in self.settings: self.settings[key] = malojaconfig[self.settings[key]] - self.authorize() + t = Thread(target=self.authorize) + t.daemon = True + t.start() + #self.authorize() # this makes sure that of every class we define, we immediately create an # instance (de facto singleton). then each instance checks if the requirements @@ -127,16 +136,6 @@ class GenericInterface: return True # per default. no authorization is necessary - # wrapper method - def request(self,url,data,responsetype): - response = urllib.request.urlopen( - url, - data=utf(data) - ) - responsedata = response.read() - if responsetype == "xml": - data = ElementTree.fromstring(responsedata) - return data # proxy scrobbler class ProxyScrobbleInterface(GenericInterface,abstract=True): @@ -155,11 +154,15 @@ class ProxyScrobbleInterface(GenericInterface,abstract=True): ) def scrobble(self,artists,title,timestamp): - response = urllib.request.urlopen( - self.proxyscrobble["scrobbleurl"], - data=utf(self.proxyscrobble_postdata(artists,title,timestamp))) - responsedata = response.read() + response = requests.post( + url=self.proxyscrobble["scrobbleurl"], + data=self.proxyscrobble_postdata(artists,title,timestamp), + headers={ + "User-Agent":self.useragent + } + ) if self.proxyscrobble["response_type"] == "xml": + responsedata = response.text data = ElementTree.fromstring(responsedata) return self.proxyscrobble_parse_response(data) @@ -211,13 +214,15 @@ class MetadataInterface(GenericInterface,abstract=True): artists, title = track artiststring = urllib.parse.quote(", ".join(artists)) titlestring = urllib.parse.quote(title) - response = urllib.request.urlopen( - self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings) + response = requests.get( + self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_track(data) else: imgurl = None @@ -227,13 +232,15 @@ class MetadataInterface(GenericInterface,abstract=True): def get_image_artist(self,artist): artiststring = urllib.parse.quote(artist) - response = urllib.request.urlopen( - self.metadata["artisturl"].format(artist=artiststring,**self.settings) + response = requests.get( + self.metadata["artisturl"].format(artist=artiststring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_artist(data) else: imgurl = None @@ -245,13 +252,15 @@ class MetadataInterface(GenericInterface,abstract=True): artists, title = album artiststring = urllib.parse.quote(", ".join(artists or [])) titlestring = urllib.parse.quote(title) - response = urllib.request.urlopen( - self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings) + response = requests.get( + self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_album(data) else: imgurl = None diff --git a/maloja/thirdparty/audiodb.py b/maloja/thirdparty/audiodb.py index 9a4d84a..5b9e71f 100644 --- a/maloja/thirdparty/audiodb.py +++ b/maloja/thirdparty/audiodb.py @@ -16,6 +16,7 @@ class AudioDB(MetadataInterface): #"response_parse_tree_track": ["tracks",0,"astrArtistThumb"], "response_parse_tree_artist": ["artists",0,"strArtistThumb"], "required_settings": ["api_key"], + "enabled_entity_types": ["artist"] } def get_image_track(self,track): diff --git a/maloja/thirdparty/deezer.py b/maloja/thirdparty/deezer.py index dfac7a8..f30a9ba 100644 --- a/maloja/thirdparty/deezer.py +++ b/maloja/thirdparty/deezer.py @@ -17,6 +17,7 @@ class Deezer(MetadataInterface): "response_parse_tree_artist": ["data",0,"artist","picture_medium"], "response_parse_tree_album": ["data",0,"album","cover_medium"], "required_settings": [], + "enabled_entity_types": ["artist","album"] } delay = 1 diff --git a/maloja/thirdparty/lastfm.py b/maloja/thirdparty/lastfm.py index e565e5f..897fef3 100644 --- a/maloja/thirdparty/lastfm.py +++ b/maloja/thirdparty/lastfm.py @@ -1,6 +1,7 @@ from . import MetadataInterface, ProxyScrobbleInterface, utf import hashlib -import urllib.parse, urllib.request +import requests +import xml.etree.ElementTree as ElementTree from doreah.logging import log class LastFM(MetadataInterface, ProxyScrobbleInterface): @@ -31,6 +32,7 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface): #"response_parse_tree_artist": ["artist","image",-1,"#text"], "response_parse_tree_album": ["album","image",-1,"#text"], "required_settings": ["apikey"], + "enabled_entity_types": ["track","album"] } def get_image_artist(self,artist): @@ -53,28 +55,39 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface): }) def authorize(self): - try: - result = self.request( - self.proxyscrobble['scrobbleurl'], - self.query_compose({ - "method":"auth.getMobileSession", - "username":self.settings["username"], - "password":self.settings["password"], - "api_key":self.settings["apikey"] - }), - responsetype="xml" - ) - self.settings["sk"] = result.find("session").findtext("key") - except Exception as e: - pass - #log("Error while authenticating with LastFM: " + repr(e)) + if all(self.settings[key] not in [None,"ASK",False] for key in ["username","password","apikey","secret"]): + try: + response = requests.post( + url=self.proxyscrobble['scrobbleurl'], + params=self.query_compose({ + "method":"auth.getMobileSession", + "username":self.settings["username"], + "password":self.settings["password"], + "api_key":self.settings["apikey"] + }), + headers={ + "User-Agent":self.useragent + } + ) + + data = ElementTree.fromstring(response.text) + self.settings["sk"] = data.find("session").findtext("key") + except Exception as e: + log("Error while authenticating with LastFM: " + repr(e)) - # creates signature and returns full query string + # creates signature and returns full query def query_compose(self,parameters): m = hashlib.md5() keys = sorted(str(k) for k in parameters) m.update(utf("".join(str(k) + str(parameters[k]) for k in keys))) m.update(utf(self.settings["secret"])) sig = m.hexdigest() - return urllib.parse.urlencode(parameters) + "&api_sig=" + sig + return {**parameters,"api_sig":sig} + + def handle_json_result_error(self,result): + if "track" in result and not result.get("track").get('album',{}): + return True + + if "error" in result and result.get("error") == 6: + return True diff --git a/maloja/thirdparty/maloja.py b/maloja/thirdparty/maloja.py index 53413f8..4975400 100644 --- a/maloja/thirdparty/maloja.py +++ b/maloja/thirdparty/maloja.py @@ -1,5 +1,5 @@ from . import ProxyScrobbleInterface, ImportInterface -import urllib.request +import requests from doreah.logging import log import json @@ -32,8 +32,8 @@ class OtherMalojaInstance(ProxyScrobbleInterface, ImportInterface): def get_remote_scrobbles(self): url = f"{self.settings['instance']}/apis/mlj_1/scrobbles" - response = urllib.request.urlopen(url) - data = json.loads(response.read().decode('utf-8')) + response = requests.get(url) + data = response.json() for scrobble in data['list']: yield scrobble diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index f16229b..668f7f3 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -1,9 +1,7 @@ from . import MetadataInterface -import urllib.parse, urllib.request -import json +import requests import time import threading -from ..__pkginfo__ import USER_AGENT class MusicBrainz(MetadataInterface): name = "MusicBrainz" @@ -11,15 +9,17 @@ class MusicBrainz(MetadataInterface): # musicbrainz is rate-limited lock = threading.Lock() - useragent = USER_AGENT + + + thumbnailsize_order = ['500','large','1200','250','small'] settings = { } metadata = { "response_type":"json", - "response_parse_tree_track": ["images",0,"thumbnails","500"], "required_settings": [], + "enabled_entity_types": ["album","track"] } def get_image_artist(self,artist): @@ -27,37 +27,105 @@ class MusicBrainz(MetadataInterface): # not supported def get_image_album(self,album): - return None - - def get_image_track(self,track): self.lock.acquire() try: - artists, title = track - artiststring = ", ".join(artists) #Join artists collection into string - titlestring = title - querystr = urllib.parse.urlencode({ - "fmt":"json", - "query":"{title} {artist}".format(artist=artiststring,title=titlestring) - }) - req = urllib.request.Request(**{ - "url":"https://musicbrainz.org/ws/2/release?" + querystr, - "method":"GET", + artists, title = album + searchstr = f'release:"{title}"' + for artist in artists: + searchstr += f' artist:"{artist}"' + res = requests.get(**{ + "url":"https://musicbrainz.org/ws/2/release", + "params":{ + "fmt":"json", + "query":searchstr + }, "headers":{ "User-Agent":self.useragent } }) - response = urllib.request.urlopen(req) - responsedata = response.read() - data = json.loads(responsedata) - mbid = data["releases"][0]["id"] - response = urllib.request.urlopen( - "https://coverartarchive.org/release/{mbid}?fmt=json".format(mbid=mbid) - ) - responsedata = response.read() - data = json.loads(responsedata) - imgurl = self.metadata_parse_response_track(data) - if imgurl is not None: imgurl = self.postprocess_url(imgurl) - return imgurl + data = res.json() + entity = data["releases"][0] + coverartendpoint = "release" + while True: + mbid = entity["id"] + try: + response = requests.get( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}", + params={ + "fmt":"json" + }, + headers={ + "User-Agent":self.useragent + } + ) + data = response.json() + thumbnails = data['images'][0]['thumbnails'] + for size in self.thumbnailsize_order: + if thumbnails.get(size) is not None: + imgurl = thumbnails.get(size) + continue + except: + imgurl = None + if imgurl is None: + entity = entity["release-group"] + # this will raise an error so we don't stay in the while loop forever + coverartendpoint = "release-group" + continue + + imgurl = self.postprocess_url(imgurl) + return imgurl + + except Exception: + return None + finally: + time.sleep(2) + self.lock.release() + + def get_image_track(self,track): + self.lock.acquire() + try: + artists, title = track + searchstr = f'recording:"{title}"' + for artist in artists: + searchstr += f' artist:"{artist}"' + res = requests.get(**{ + "url":"https://musicbrainz.org/ws/2/recording", + "params":{ + "fmt":"json", + "query":searchstr + }, + "headers":{ + "User-Agent":self.useragent + } + }) + data = res.json() + entity = data["recordings"][0]["releases"][0] + coverartendpoint = "release" + while True: + mbid = entity["id"] + try: + response = requests.get( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}", + params={ + "fmt":"json" + } + ) + data = response.json() + thumbnails = data['images'][0]['thumbnails'] + for size in self.thumbnailsize_order: + if thumbnails.get(size) is not None: + imgurl = thumbnails.get(size) + continue + except: + imgurl = None + if imgurl is None: + entity = entity["release-group"] + # this will raise an error so we don't stay in the while loop forever + coverartendpoint = "release-group" + continue + + imgurl = self.postprocess_url(imgurl) + return imgurl except Exception: return None diff --git a/maloja/thirdparty/spotify.py b/maloja/thirdparty/spotify.py index 3a8e55f..93f4d62 100644 --- a/maloja/thirdparty/spotify.py +++ b/maloja/thirdparty/spotify.py @@ -1,6 +1,5 @@ from . import MetadataInterface, utf, b64 -import urllib.parse, urllib.request -import json +import requests from threading import Timer from doreah.logging import log @@ -22,6 +21,7 @@ class Spotify(MetadataInterface): "response_parse_tree_album": ["albums","items",0,"images",0,"url"], "response_parse_tree_artist": ["artists","items",0,"images",0,"url"], "required_settings": ["apiid","secret"], + "enabled_entity_types": ["artist","album","track"] } def authorize(self): @@ -31,15 +31,14 @@ class Spotify(MetadataInterface): try: keys = { "url":"https://accounts.spotify.com/api/token", - "method":"POST", "headers":{ - "Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8") + "Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8"), + "User-Agent": self.useragent }, - "data":bytes(urllib.parse.urlencode({"grant_type":"client_credentials"}),encoding="utf-8") + "data":{"grant_type":"client_credentials"} } - req = urllib.request.Request(**keys) - response = urllib.request.urlopen(req) - responsedata = json.loads(response.read()) + res = requests.post(**keys) + responsedata = res.json() if "error" in responsedata: log("Error authenticating with Spotify: " + responsedata['error_description']) expire = 3600 @@ -47,6 +46,13 @@ class Spotify(MetadataInterface): expire = responsedata.get("expires_in",3600) self.settings["token"] = responsedata["access_token"] #log("Successfully authenticated with Spotify") - Timer(expire,self.authorize).start() + t = Timer(expire,self.authorize) + t.daemon = True + t.start() except Exception as e: log("Error while authenticating with Spotify: " + repr(e)) + + def handle_json_result_error(self,result): + result = result.get('tracks') or result.get('albums') or result.get('artists') + if not result['items']: + return True \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 275e135..e543872 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "3.2.0" +version = "3.2.1" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.10"