From f9ce0e6ba9d4b709a5012d8524219d8cdaf6dbd1 Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 02:49:26 +0200 Subject: [PATCH 1/9] Fixed broken link for album top weeks --- maloja/web/jinja/partials/awards_album.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/web/jinja/partials/awards_album.jinja b/maloja/web/jinja/partials/awards_album.jinja index 5ded047..b9337a2 100644 --- a/maloja/web/jinja/partials/awards_album.jinja +++ b/maloja/web/jinja/partials/awards_album.jinja @@ -28,7 +28,7 @@ {% macro topweeks(info) %} -{% set encodedtrack = mlj_uri.uriencode({'album':info.album}) %} +{% set encodedalbum = mlj_uri.uriencode({'album':info.album}) %} From 451014f6e7674ff2f377cd13f91413d5ac45e8c3 Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 04:22:33 +0200 Subject: [PATCH 2/9] Temporary fix for scrobbling with no album info --- maloja/database/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index 48fb708..f40a85e 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -151,7 +151,7 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None): # if we send [] as albumartists, it means various # if we send nothing, the scrobbler just doesnt support it and we assume track artists - if 'album_artists' not in scrobbleinfo: + if ('album_title' in scrobbleinfo) and ('album_artists' not in scrobbleinfo): scrobbleinfo['album_artists'] = scrobbleinfo.get('track_artists') # New plan, do this further down @@ -181,6 +181,9 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None): "rawscrobble":rawscrobble } + if scrobbledict["track"]["album"]["albumtitle"] is None and scrobbledict["track"]["album"]["artists"] is None: + del scrobbledict["track"]["album"] + return scrobbledict From d7d2f676a75c884e3a85b9016aba56ab8b1da7fd Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 05:18:43 +0200 Subject: [PATCH 3/9] Removed some superfluous id resolving of entities --- maloja/database/__init__.py | 45 ++++++++++++++++++++----------------- maloja/database/sqldb.py | 25 ++++++++++----------- 2 files changed, 36 insertions(+), 34 deletions(-) diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index f40a85e..eae6634 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -181,7 +181,7 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None): "rawscrobble":rawscrobble } - if scrobbledict["track"]["album"]["albumtitle"] is None and scrobbledict["track"]["album"]["artists"] is None: + if scrobbledict["track"]["album"]["albumtitle"] is None: del scrobbledict["track"]["album"] return scrobbledict @@ -320,29 +320,29 @@ def get_albums_artist_appears_on(dbconn=None,**keys): @waitfordb -def get_charts_artists(dbconn=None,**keys): +def get_charts_artists(dbconn=None,resolve_ids=True,**keys): (since,to) = keys.get('timerange').timestamps() - result = sqldb.count_scrobbles_by_artist(since=since,to=to,dbconn=dbconn) + result = sqldb.count_scrobbles_by_artist(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn) return result @waitfordb -def get_charts_tracks(dbconn=None,**keys): +def get_charts_tracks(dbconn=None,resolve_ids=True,**keys): (since,to) = keys.get('timerange').timestamps() if 'artist' in keys: - result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],dbconn=dbconn) + result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],resolve_ids=resolve_ids,dbconn=dbconn) elif 'album' in keys: - result = sqldb.count_scrobbles_by_track_of_album(since=since,to=to,album=keys['album'],dbconn=dbconn) + result = sqldb.count_scrobbles_by_track_of_album(since=since,to=to,album=keys['album'],resolve_ids=resolve_ids,dbconn=dbconn) else: - result = sqldb.count_scrobbles_by_track(since=since,to=to,dbconn=dbconn) + result = sqldb.count_scrobbles_by_track(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn) return result @waitfordb -def get_charts_albums(dbconn=None,**keys): +def get_charts_albums(dbconn=None,resolve_ids=True,**keys): (since,to) = keys.get('timerange').timestamps() if 'artist' in keys: - result = sqldb.count_scrobbles_by_album_of_artist(since=since,to=to,artist=keys['artist'],dbconn=dbconn) + result = sqldb.count_scrobbles_by_album_of_artist(since=since,to=to,artist=keys['artist'],resolve_ids=resolve_ids,dbconn=dbconn) else: - result = sqldb.count_scrobbles_by_album(since=since,to=to,dbconn=dbconn) + result = sqldb.count_scrobbles_by_album(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn) return result @waitfordb @@ -364,29 +364,32 @@ def get_performance(dbconn=None,**keys): for rng in rngs: if "track" in keys: - track = sqldb.get_track(sqldb.get_track_id(keys['track'],dbconn=dbconn),dbconn=dbconn) - charts = get_charts_tracks(timerange=rng,dbconn=dbconn) + track_id = sqldb.get_track_id(keys['track'],dbconn=dbconn) + #track = sqldb.get_track(track_id,dbconn=dbconn) + charts = get_charts_tracks(timerange=rng,resolve_ids=False,dbconn=dbconn) rank = None for c in charts: - if c["track"] == track: + if c["track_id"] == track_id: rank = c["rank"] break elif "artist" in keys: - artist = sqldb.get_artist(sqldb.get_artist_id(keys['artist'],dbconn=dbconn),dbconn=dbconn) + artist_id = sqldb.get_artist_id(keys['artist'],dbconn=dbconn) + #artist = sqldb.get_artist(artist_id,dbconn=dbconn) # ^this is the most useless line in programming history # but I like consistency - charts = get_charts_artists(timerange=rng,dbconn=dbconn) + charts = get_charts_artists(timerange=rng,resolve_ids=False,dbconn=dbconn) rank = None for c in charts: - if c["artist"] == artist: + if c["artist_id"] == artist_id: rank = c["rank"] break elif "album" in keys: - album = sqldb.get_album(sqldb.get_album_id(keys['album'],dbconn=dbconn),dbconn=dbconn) - charts = get_charts_albums(timerange=rng,dbconn=dbconn) + album_id = sqldb.get_album_id(keys['album'],dbconn=dbconn) + #album = sqldb.get_album(album_id,dbconn=dbconn) + charts = get_charts_albums(timerange=rng,resolve_ids=False,dbconn=dbconn) rank = None for c in charts: - if c["album"] == album: + if c["album_id"] == album_id: rank = c["rank"] break else: @@ -502,10 +505,10 @@ def track_info(dbconn=None,**keys): track_id = sqldb.get_track_id(track,dbconn=dbconn) track = sqldb.get_track(track_id,dbconn=dbconn) - alltimecharts = get_charts_tracks(timerange=alltime(),dbconn=dbconn) + alltimecharts = get_charts_tracks(timerange=alltime(),resolve_ids=False,dbconn=dbconn) #scrobbles = get_scrobbles_num(track=track,timerange=alltime()) - c = [e for e in alltimecharts if e["track"] == track][0] + c = [e for e in alltimecharts if e["track_id"] == track_id][0] scrobbles = c["scrobbles"] position = c["rank"] cert = None diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index 0497456..95f5841 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -866,7 +866,6 @@ def count_scrobbles_by_artist(since,to,resolve_ids=True,dbconn=None): result = dbconn.execute(op).all() if resolve_ids: - counts = [row.count for row in result] artists = get_artists_map([row.artist_id for row in result],dbconn=dbconn) result = [{'scrobbles':row.count,'artist':artists[row.artist_id]} for row in result] else: @@ -889,7 +888,6 @@ def count_scrobbles_by_track(since,to,resolve_ids=True,dbconn=None): result = dbconn.execute(op).all() if resolve_ids: - counts = [row.count for row in result] tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] else: @@ -918,7 +916,6 @@ def count_scrobbles_by_album(since,to,resolve_ids=True,dbconn=None): result = dbconn.execute(op).all() if resolve_ids: - counts = [row.count for row in result] albums = get_albums_map([row.album_id for row in result],dbconn=dbconn) result = [{'scrobbles':row.count,'album':albums[row.album_id]} for row in result] else: @@ -956,7 +953,6 @@ def count_scrobbles_by_album_of_artist(since,to,artist,resolve_ids=True,dbconn=N result = dbconn.execute(op).all() if resolve_ids: - counts = [row.count for row in result] albums = get_albums_map([row.album_id for row in result],dbconn=dbconn) result = [{'scrobbles':row.count,'album':albums[row.album_id]} for row in result] else: @@ -994,7 +990,6 @@ def count_scrobbles_of_artist_by_album(since,to,artist,resolve_ids=True,dbconn=N result = dbconn.execute(op).all() if resolve_ids: - counts = [row.count for row in result] albums = get_albums_map([row.album_id for row in result],dbconn=dbconn) result = [{'scrobbles':row.count,'album':albums[row.album_id]} for row in result] else: @@ -1005,7 +1000,7 @@ def count_scrobbles_of_artist_by_album(since,to,artist,resolve_ids=True,dbconn=N @cached_wrapper @connection_provider -def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None): +def count_scrobbles_by_track_of_artist(since,to,artist,resolve_ids=True,dbconn=None): artist_id = get_artist_id(artist,dbconn=dbconn) @@ -1026,16 +1021,18 @@ def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None): result = dbconn.execute(op).all() - counts = [row.count for row in result] - tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) - result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + if resolve_ids: + tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + else: + result = [{'scrobbles':row.count,'track_id':row.track_id} for row in result] result = rank(result,key='scrobbles') return result @cached_wrapper @connection_provider -def count_scrobbles_by_track_of_album(since,to,album,dbconn=None): +def count_scrobbles_by_track_of_album(since,to,album,resolve_ids=True,dbconn=None): album_id = get_album_id(album,dbconn=dbconn) @@ -1056,9 +1053,11 @@ def count_scrobbles_by_track_of_album(since,to,album,dbconn=None): result = dbconn.execute(op).all() - counts = [row.count for row in result] - tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) - result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + if resolve_ids: + tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + else: + result = [{'scrobbles':row.count,'track_id':row.track_id} for row in result] result = rank(result,key='scrobbles') return result From 246608f5e09d948164c114985b17f3c1b41b6984 Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 05:33:30 +0200 Subject: [PATCH 4/9] Album updates now properly evict caches --- maloja/database/sqldb.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index 95f5841..f511488 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -6,7 +6,7 @@ from datetime import datetime from threading import Lock from ..pkg_global.conf import data_dir -from .dbcache import cached_wrapper, cached_wrapper_individual +from .dbcache import cached_wrapper, cached_wrapper_individual, invalidate_caches, invalidate_entity_cache from . import exceptions as exc from doreah.logging import log @@ -352,6 +352,11 @@ def add_track_to_album(track_id,album_id,replace=False,dbconn=None): ) result = dbconn.execute(op) + + invalidate_entity_cache() # because album info has changed + invalidate_caches() # changing album info of tracks will change album charts + + return True @connection_provider From 19de87cb66656350ed35cfc25256849ae4c1790d Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 14:10:53 +0200 Subject: [PATCH 5/9] Fixed ephemeral entity creation --- maloja/database/__init__.py | 15 ++++++++++----- maloja/database/exceptions.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index eae6634..0995496 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -110,7 +110,7 @@ def incoming_scrobble(rawscrobble,fix=True,client=None,api=None,dbconn=None): proxy_scrobble_all(scrobbledict['track']['artists'],scrobbledict['track']['title'],scrobbledict['time']) dbcache.invalidate_caches(scrobbledict['time']) - dbcache.invalidate_entity_cache() # because album info might have changed + #return {"status":"success","scrobble":scrobbledict} return scrobbledict @@ -450,7 +450,9 @@ def artist_info(dbconn=None,**keys): artist = keys.get('artist') if artist is None: raise exceptions.MissingEntityParameter() - artist_id = sqldb.get_artist_id(artist,dbconn=dbconn) + artist_id = sqldb.get_artist_id(artist,create_new=False,dbconn=dbconn) + if not artist_id: raise exceptions.ArtistDoesNotExist(artist) + artist = sqldb.get_artist(artist_id,dbconn=dbconn) alltimecharts = get_charts_artists(timerange=alltime(),dbconn=dbconn) #we cant take the scrobble number from the charts because that includes all countas scrobbles @@ -503,7 +505,9 @@ def track_info(dbconn=None,**keys): track = keys.get('track') if track is None: raise exceptions.MissingEntityParameter() - track_id = sqldb.get_track_id(track,dbconn=dbconn) + track_id = sqldb.get_track_id(track,create_new=False,dbconn=dbconn) + if not track_id: raise exceptions.TrackDoesNotExist(track['title']) + track = sqldb.get_track(track_id,dbconn=dbconn) alltimecharts = get_charts_tracks(timerange=alltime(),resolve_ids=False,dbconn=dbconn) #scrobbles = get_scrobbles_num(track=track,timerange=alltime()) @@ -539,9 +543,10 @@ def album_info(dbconn=None,**keys): album = keys.get('album') if album is None: raise exceptions.MissingEntityParameter() - album_id = sqldb.get_album_id(album,dbconn=dbconn) - album = sqldb.get_album(album_id,dbconn=dbconn) + album_id = sqldb.get_album_id(album,create_new=False,dbconn=dbconn) + if not album_id: raise exceptions.AlbumDoesNotExist(album['albumtitle']) + album = sqldb.get_album(album_id,dbconn=dbconn) alltimecharts = get_charts_albums(timerange=alltime(),dbconn=dbconn) #scrobbles = get_scrobbles_num(track=track,timerange=alltime()) diff --git a/maloja/database/exceptions.py b/maloja/database/exceptions.py index a2dadd9..0d4fce4 100644 --- a/maloja/database/exceptions.py +++ b/maloja/database/exceptions.py @@ -27,3 +27,19 @@ class MissingScrobbleParameters(Exception): class MissingEntityParameter(Exception): pass + +class EntityDoesNotExist(HTTPError): + entitytype = 'Entity' + def __init__(self,name): + self.entityname = name + super().__init__( + status=404, + body=f"The {self.entitytype} '{self.entityname}' does not exist in the database." + ) + +class ArtistDoesNotExist(EntityDoesNotExist): + entitytype = 'Artist' +class AlbumDoesNotExist(EntityDoesNotExist): + entitytype = 'Album' +class TrackDoesNotExist(EntityDoesNotExist): + entitytype = 'Track' From 55363bf31b86e7862f9ae9a6d4166bf4ecf8f73d Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 14:42:58 +0200 Subject: [PATCH 6/9] Disabled more maintenance nonsense when running tasks --- maloja/database/dbcache.py | 19 +++++++++-- maloja/database/sqldb.py | 64 ++++++++++++++++++++------------------ 2 files changed, 50 insertions(+), 33 deletions(-) diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index 582a9e7..0bd0c9a 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -23,11 +23,13 @@ if malojaconfig['USE_GLOBAL_CACHE']: @runhourly def maintenance(): from . import AUX_MODE - if not AUX_MODE: - print_stats() - trim_cache() + if AUX_MODE: return + print_stats() + trim_cache() def print_stats(): + from . import AUX_MODE + if AUX_MODE: return for name,c in (('Cache',cache),('Entity Cache',entitycache)): hits, misses = c.get_stats() log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}") @@ -35,6 +37,8 @@ if malojaconfig['USE_GLOBAL_CACHE']: def cached_wrapper(inner_func): + from . import AUX_MODE + if AUX_MODE: return inner_func def outer_func(*args,**kwargs): @@ -59,6 +63,8 @@ if malojaconfig['USE_GLOBAL_CACHE']: # we don't want a new cache entry for every single combination, but keep a common # cache that's aware of what we're calling def cached_wrapper_individual(inner_func): + from . import AUX_MODE + if AUX_MODE: return def outer_func(set_arg,**kwargs): if 'dbconn' in kwargs: @@ -83,6 +89,9 @@ if malojaconfig['USE_GLOBAL_CACHE']: return outer_func def invalidate_caches(scrobbletime=None): + from . import AUX_MODE + if AUX_MODE: return + cleared, kept = 0, 0 for k in cache.keys(): # VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'! @@ -95,10 +104,14 @@ if malojaconfig['USE_GLOBAL_CACHE']: def invalidate_entity_cache(): + from . import AUX_MODE + if AUX_MODE: return entitycache.clear() def trim_cache(): + from . import AUX_MODE + if AUX_MODE: return ramprct = psutil.virtual_memory().percent if ramprct > malojaconfig["DB_MAX_MEMORY"]: log(f"{ramprct}% RAM usage, clearing cache!") diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index f511488..c39ba5a 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -1360,40 +1360,40 @@ def search_album(searchterm,dbconn=None): def clean_db(dbconn=None): from . import AUX_MODE + if AUX_MODE: return - if not AUX_MODE: - with SCROBBLE_LOCK: - log(f"Database Cleanup...") + with SCROBBLE_LOCK: + log(f"Database Cleanup...") - to_delete = [ - # tracks with no scrobbles (trackartist entries first) - "from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))", - "from tracks where id not in (select track_id from scrobbles)", - # artists with no tracks AND no albums - "from artists where id not in (select artist_id from trackartists) \ - and id not in (select target_artist from associated_artists) \ - and id not in (select artist_id from albumartists)", - # tracks with no artists (scrobbles first) - "from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))", - "from tracks where id not in (select track_id from trackartists)", - # albums with no tracks (albumartist entries first) - "from albumartists where album_id in (select id from albums where id not in (select album_id from tracks where album_id is not null))", - "from albums where id not in (select album_id from tracks where album_id is not null)", - # albumartist entries that are missing a reference - "from albumartists where album_id not in (select album_id from tracks where album_id is not null)", - "from albumartists where artist_id not in (select id from artists)", - # trackartist entries that mare missing a reference - "from trackartists where track_id not in (select id from tracks)", - "from trackartists where artist_id not in (select id from artists)" - ] + to_delete = [ + # tracks with no scrobbles (trackartist entries first) + "from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))", + "from tracks where id not in (select track_id from scrobbles)", + # artists with no tracks AND no albums + "from artists where id not in (select artist_id from trackartists) \ + and id not in (select target_artist from associated_artists) \ + and id not in (select artist_id from albumartists)", + # tracks with no artists (scrobbles first) + "from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))", + "from tracks where id not in (select track_id from trackartists)", + # albums with no tracks (albumartist entries first) + "from albumartists where album_id in (select id from albums where id not in (select album_id from tracks where album_id is not null))", + "from albums where id not in (select album_id from tracks where album_id is not null)", + # albumartist entries that are missing a reference + "from albumartists where album_id not in (select album_id from tracks where album_id is not null)", + "from albumartists where artist_id not in (select id from artists)", + # trackartist entries that mare missing a reference + "from trackartists where track_id not in (select id from tracks)", + "from trackartists where artist_id not in (select id from artists)" + ] - for d in to_delete: - selection = dbconn.execute(sql.text(f"select * {d}")) - for row in selection.all(): - log(f"Deleting {row}") - deletion = dbconn.execute(sql.text(f"delete {d}")) + for d in to_delete: + selection = dbconn.execute(sql.text(f"select * {d}")) + for row in selection.all(): + log(f"Deleting {row}") + deletion = dbconn.execute(sql.text(f"delete {d}")) - log("Database Cleanup complete!") + log("Database Cleanup complete!") @@ -1407,6 +1407,10 @@ def clean_db(dbconn=None): @runmonthly def renormalize_names(): + + from . import AUX_MODE + if AUX_MODE: return + with SCROBBLE_LOCK: with engine.begin() as conn: rows = conn.execute(DB['artists'].select()).all() From 924d4718db8f1bf3c91733a2160dd33cf87e59b4 Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 14:47:30 +0200 Subject: [PATCH 7/9] Fixed album parsing from raw scrobble, GH-207 --- maloja/database/sqldb.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index c39ba5a..b6fd6e1 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -1513,7 +1513,7 @@ def guess_albums(track_ids=None,replace=False,dbconn=None): # get all scrobbles of the respective tracks that have some info conditions = [ - DB['scrobbles'].c.extra.isnot(None) + DB['scrobbles'].c.extra.isnot(None) | DB['scrobbles'].c.rawscrobble.isnot(None) ] if track_ids is not None: # only do these tracks @@ -1537,10 +1537,13 @@ def guess_albums(track_ids=None,replace=False,dbconn=None): # for each track, count what album info appears how often possible_albums = {} for row in result: - extrainfo = json.loads(row.extra) - albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title") - albumartists = extrainfo.get("album_artists",[]) + albumtitle, albumartists = None, None + if row.extra: + extrainfo = json.loads(row.extra) + albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title") + albumartists = extrainfo.get("album_artists",[]) if not albumtitle: + # either we didn't have info in the exta col, or there was no albumtitle # try the raw scrobble extrainfo = json.loads(row.rawscrobble) albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title") From 2c44745abcc5dffe0ef6e1b28d74ee6ec422e5c4 Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 14:50:53 +0200 Subject: [PATCH 8/9] This is getting worse and worse --- maloja/database/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index 0995496..184a437 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -1,6 +1,17 @@ # server from bottle import request, response, FormsDict + +# we're running an auxiliary task that doesn't require all the random background +# nonsense to be fired up +# this is temporary +# FIX YO DAMN ARCHITECTURE ALREADY +AUX_MODE = False +def set_aux_mode(): + global AUX_MODE + AUX_MODE = True + + # rest of the project from ..cleanup import CleanerAgent from .. import images @@ -45,14 +56,7 @@ dbstatus = { } -# we're running an auxiliary task that doesn't require all the random background -# nonsense to be fired up -# this is temporary -# FIX YO DAMN ARCHITECTURE ALREADY -AUX_MODE = False -def set_aux_mode(): - global AUX_MODE - AUX_MODE = True + From 0ba55d466dc967418c4555df72b399683fa91e92 Mon Sep 17 00:00:00 2001 From: krateng Date: Fri, 31 Mar 2023 14:59:54 +0200 Subject: [PATCH 9/9] Fixed more read-only id queries --- maloja/database/sqldb.py | 6 +++++- maloja/images.py | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index b6fd6e1..f64c43e 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -396,17 +396,21 @@ def get_track_id(trackdict,create_new=True,update_album=False,dbconn=None): #print("required artists",artist_ids,"this match",match_artist_ids) if set(artist_ids) == set(match_artist_ids): #print("ID for",trackdict['title'],"was",row[0]) - if trackdict.get('album'): + if trackdict.get('album') and create_new: + # if we don't supply create_new, it means we just want to get info about a track + # which means no need to write album info, even if it was new add_track_to_album(row.id,get_album_id(trackdict['album'],dbconn=dbconn),replace=update_album,dbconn=dbconn) return row.id if not create_new: return None + print("Creating new track") op = DB['tracks'].insert().values( **track_dict_to_db(trackdict,dbconn=dbconn) ) result = dbconn.execute(op) track_id = result.inserted_primary_key[0] + print(track_id) for artist_id in artist_ids: op = DB['trackartists'].insert().values( diff --git a/maloja/images.py b/maloja/images.py index 4c2fa22..f4425e4 100644 --- a/maloja/images.py +++ b/maloja/images.py @@ -111,20 +111,20 @@ def dl_image(url): ### even if we have already cached it, we will handle that on request def get_track_image(track=None,track_id=None): if track_id is None: - track_id = database.sqldb.get_track_id(track) + track_id = database.sqldb.get_track_id(track,create_new=False) return f"/image?type=track&id={track_id}" def get_artist_image(artist=None,artist_id=None): if artist_id is None: - artist_id = database.sqldb.get_artist_id(artist) + artist_id = database.sqldb.get_artist_id(artist,create_new=False) return f"/image?type=artist&id={artist_id}" def get_album_image(album=None,album_id=None): if album_id is None: - album_id = database.sqldb.get_album_id(album) + album_id = database.sqldb.get_album_id(album,create_new=False) return f"/image?type=album&id={album_id}"