From 0f39ecbf7e99d5cf3ce1cde3a45d899810e5140f Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 12 Apr 2022 16:39:44 +0200 Subject: [PATCH 01/12] Fixed funding file --- .github/FUNDING.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 393f47d..0fd9451 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1 @@ -custom: ["https://flattr.com/@Krateng", "https://paypal.me/krateng", "bitcoin:1krat8JMniJBTiHftMfR1LtF3Y1w5DAxx"] +custom: ["https://flattr.com/@Krateng", "https://paypal.me/krateng"] From 50cf592a75b9bf9a4a54e580ceefabcd49a25e17 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 12 Apr 2022 17:57:13 +0200 Subject: [PATCH 02/12] Fixed upgrading early scrobbles, GH-106 --- maloja/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/upgrade.py b/maloja/upgrade.py index 7dff80e..e9abaab 100644 --- a/maloja/upgrade.py +++ b/maloja/upgrade.py @@ -67,7 +67,7 @@ def upgrade_db(callback_add_scrobbles): scrobblelist = [] log(f"\tImporting from {sf} ({idx}/{len(scrobblefiles)}) - {len(scrobbles)} Scrobbles") for scrobble in scrobbles: - timestamp, artists, title, album, duration = scrobble + timestamp, artists, title, album, duration, *_ = scrobble + [None,None] if album in ('-',''): album = None if duration in ('-',''): duration = None scrobblelist.append({ From 700b81217cb585df631d6f069243c56074cd1b71 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 12 Apr 2022 17:59:32 +0200 Subject: [PATCH 03/12] Version bump --- maloja/__pkginfo__.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index f772f7e..079717b 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "3.0.0" +VERSION = "3.0.1" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/pyproject.toml b/pyproject.toml index d9f2fb4..5b70214 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "3.0.0" +version = "3.0.1" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.6" From 6b39ca8b1993f6a4efd22b326746ec395421de01 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 12 Apr 2022 19:03:55 +0200 Subject: [PATCH 04/12] Added duration warning to upgrade page --- maloja/database/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index ac3ec86..f37caa4 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -46,7 +46,7 @@ class DatabaseNotBuilt(HTTPError): def __init__(self): super().__init__( status=503, - body="The Maloja Database is being upgraded to Version 3. This could take several minutes.", + body="The Maloja Database is being upgraded to Version 3. This could take quite a long time! (~ 2-5 minutes per 10 000 scrobbles)", headers={"Retry-After":120} ) From 1c2062c5129e8d017a6a93c4545934f379aa14f2 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 12 Apr 2022 19:04:22 +0200 Subject: [PATCH 05/12] Reenabled site generation profiling --- maloja/proccontrol/profiler.py | 24 +++++++++++++++--------- maloja/server.py | 1 + 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/maloja/proccontrol/profiler.py b/maloja/proccontrol/profiler.py index 3a98d35..068bf61 100644 --- a/maloja/proccontrol/profiler.py +++ b/maloja/proccontrol/profiler.py @@ -11,24 +11,30 @@ from ..globalconf import data_dir profiler = cProfile.Profile() +FULL_PROFILE = False + def profile(func): def newfunc(*args,**kwargs): - benchmarkfolder = data_dir['logs']("benchmarks") - os.makedirs(benchmarkfolder,exist_ok=True) + if FULL_PROFILE: + benchmarkfolder = data_dir['logs']("benchmarks") + os.makedirs(benchmarkfolder,exist_ok=True) clock = Clock() clock.start() - profiler.enable() + if FULL_PROFILE: + profiler.enable() result = func(*args,**kwargs) - profiler.disable() - + if FULL_PROFILE: + profiler.disable() + log(f"Executed {func.__name__} ({args}, {kwargs}) in {clock.stop():.2f}s",module="debug_performance") - try: - pstats.Stats(profiler).dump_stats(os.path.join(benchmarkfolder,f"{func.__name__}.stats")) - except: - pass + if FULL_PROFILE: + try: + pstats.Stats(profiler).dump_stats(os.path.join(benchmarkfolder,f"{func.__name__}.stats")) + except: + pass return result diff --git a/maloja/server.py b/maloja/server.py index 5975a39..639d20b 100644 --- a/maloja/server.py +++ b/maloja/server.py @@ -236,6 +236,7 @@ def static(path): ### DYNAMIC +@profile def jinja_page(name): if name in aliases: redirect(aliases[name]) keys = remove_identical(FormsDict.decode(request.query)) From dad136562750395840f1eae6690739b12329e550 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 12 Apr 2022 20:33:36 +0200 Subject: [PATCH 06/12] Renamed imported scrobble folder --- maloja/upgrade.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/upgrade.py b/maloja/upgrade.py index e9abaab..ca84001 100644 --- a/maloja/upgrade.py +++ b/maloja/upgrade.py @@ -44,7 +44,7 @@ def upgrade_apikeys(): def upgrade_db(callback_add_scrobbles): oldfolder = os.path.join(dir_settings['state'],"scrobbles") - newfolder = os.path.join(dir_settings['state'],".oldscrobbles") + newfolder = os.path.join(dir_settings['state'],".v2scrobbles") os.makedirs(newfolder,exist_ok=True) if os.path.exists(oldfolder): scrobblefiles = [f for f in os.listdir(oldfolder) if f.endswith(".tsv")] From ef312c6ba9010d8d647a86457a65b917bc5427a0 Mon Sep 17 00:00:00 2001 From: krateng Date: Tue, 12 Apr 2022 23:04:17 +0200 Subject: [PATCH 07/12] Updated admin mode info --- maloja/web/jinja/admin_overview.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/maloja/web/jinja/admin_overview.jinja b/maloja/web/jinja/admin_overview.jinja index a0fc2f8..36993a6 100644 --- a/maloja/web/jinja/admin_overview.jinja +++ b/maloja/web/jinja/admin_overview.jinja @@ -65,6 +65,7 @@ With Admin Mode activated, you can:
  • manually scrobble from track pages
    • +
  • delete scrobbles
  • upload artist and track art by dropping a file on the existing image on an artist or track page
  • see more detailed error pages
From 42579ad1f0cb4bcef6e4b053a9f97863d3d8584e Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 13 Apr 2022 15:42:45 +0200 Subject: [PATCH 08/12] Fixed missing import, GH-108 --- maloja/apis/native_v1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maloja/apis/native_v1.py b/maloja/apis/native_v1.py index 6e5ccac..5bab8c0 100644 --- a/maloja/apis/native_v1.py +++ b/maloja/apis/native_v1.py @@ -1,4 +1,5 @@ import os +import math from bottle import response, static_file, request, FormsDict From 4a8221f7a08f679b21c1fb619f03e5f922a1dc2b Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 13 Apr 2022 17:51:17 +0200 Subject: [PATCH 09/12] Added waitress warning interception --- maloja/__pkginfo__.py | 2 +- maloja/server.py | 29 ++++++++++++++++++++++++++++- pyproject.toml | 2 +- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 079717b..ab4d615 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "3.0.1" +VERSION = "3.0.2" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/maloja/server.py b/maloja/server.py index 639d20b..4796838 100644 --- a/maloja/server.py +++ b/maloja/server.py @@ -6,6 +6,7 @@ import setproctitle from importlib import resources from css_html_js_minify import html_minify, css_minify import datauri +import time # server stuff @@ -35,7 +36,7 @@ from .proccontrol.profiler import profile PORT = malojaconfig["PORT"] HOST = malojaconfig["HOST"] -THREADS = 12 +THREADS = 16 BaseRequest.MEMFILE_MAX = 15 * 1024 * 1024 #STATICFOLDER = importlib.resources.path(__name__,"web/static") @@ -296,6 +297,31 @@ def redirect_track(artists,title): ##### +# warning interception +import logging + +class WaitressLogHandler(): + def __init__(self): + self.lastwarned = 0 + self.barrier = 5 + self.level = 20 + self.filters = [] + def handle(self,record): + if record.name == 'waitress.queue': + now = time.time() + depth = record.args[0] + + if depth > self.barrier: + log(f"Waitress Task Queue Depth at {depth}") + self.lastwarned = now + self.barrier = max(depth,self.barrier+5) + elif now - self.lastwarned > 5: + self.barrier = max(5,self.barrier-5) + else: + log(f"Waitress: {record.msg % record.args}") +logging.getLogger().addHandler(WaitressLogHandler()) + + def run_server(): log("Starting up Maloja server...") @@ -303,6 +329,7 @@ def run_server(): Thread(target=database.start_db).start() + try: #run(webserver, host=HOST, port=MAIN_PORT, server='waitress') log(f"Listening on {HOST}:{PORT}") diff --git a/pyproject.toml b/pyproject.toml index 5b70214..653cc47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "3.0.1" +version = "3.0.2" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.6" From c076518d76486910029b7b5fbb6700c09875613d Mon Sep 17 00:00:00 2001 From: Daniel Aleksandersen Date: Thu, 14 Apr 2022 13:02:21 +0200 Subject: [PATCH 10/12] Set Referrer-Policy to same-origin Remove the Referer (sic) HTTP request header from external requests (e.g. to the image CDNs). The charset directive must be included in the first TCP packet. It should be set at the very top of the document. Grouping document mode metas and descriptive metadata in separate groups. --- maloja/web/jinja/abstracts/base.jinja | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/maloja/web/jinja/abstracts/base.jinja b/maloja/web/jinja/abstracts/base.jinja index 29cb70d..8e1645c 100644 --- a/maloja/web/jinja/abstracts/base.jinja +++ b/maloja/web/jinja/abstracts/base.jinja @@ -3,8 +3,10 @@ - {% block title %}{% endblock %} + + + {% block title %}{% endblock %} From 9d8752d052dcda270d286d0797bcfd265c23ae9f Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 14 Apr 2022 14:49:59 +0200 Subject: [PATCH 11/12] Fixed proper recognition of artist and track entities, fix GH-111 --- maloja/web/jinja/snippets/entityrow.jinja | 4 ++-- maloja/web/jinja/snippets/links.jinja | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/maloja/web/jinja/snippets/entityrow.jinja b/maloja/web/jinja/snippets/entityrow.jinja index e44f7fd..58fc46c 100644 --- a/maloja/web/jinja/snippets/entityrow.jinja +++ b/maloja/web/jinja/snippets/entityrow.jinja @@ -2,14 +2,14 @@ {% import 'snippets/links.jinja' as links %} -{% if 'artists' in entity %} +{% if entity is mapping and 'artists' in entity %} {% set img = images.get_track_image(entity) %} {% else %} {% set img = images.get_artist_image(entity) %} {% endif %}
-{% if "artists" in entity %} +{% if entity is mapping and 'artists' in entity %} {% if settings['TRACK_SEARCH_PROVIDER'] %} {{ links.link_search(entity) }} {% endif %} diff --git a/maloja/web/jinja/snippets/links.jinja b/maloja/web/jinja/snippets/links.jinja index f66bd3b..bb014e1 100644 --- a/maloja/web/jinja/snippets/links.jinja +++ b/maloja/web/jinja/snippets/links.jinja @@ -1,5 +1,5 @@ {% macro link(entity) -%} - {% if 'artists' in entity %} + {% if entity is mapping and 'artists' in entity %} {% set name = entity.title %} {% else %} {% set name = entity %} @@ -17,7 +17,7 @@ {% macro url(entity) %} - {% if 'artists' in entity -%} + {% if entity is mapping and 'artists' in entity -%} {{ mlj_uri.create_uri("/track",{'track':entity}) }} {%- else -%} {{ mlj_uri.create_uri("/artist",{'artist':entity}) }} From bcb1d36b4ab4d08806e86ebf45a63f8730b4121d Mon Sep 17 00:00:00 2001 From: krateng Date: Thu, 14 Apr 2022 15:10:15 +0200 Subject: [PATCH 12/12] Exit codes for main function, fix GH-113 --- maloja/proccontrol/control.py | 5 ++--- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py index fd2aeba..691e3c0 100644 --- a/maloja/proccontrol/control.py +++ b/maloja/proccontrol/control.py @@ -128,14 +128,13 @@ def main(*args,**kwargs): if "version" in kwargs: print(info.VERSION) + return True else: try: action, *args = args action = actions[action] except (ValueError, KeyError): print("Valid commands: " + " ".join(a for a in actions)) - return + return False return action(*args,**kwargs) - - return True diff --git a/pyproject.toml b/pyproject.toml index 653cc47..d9600f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ classifiers = [ dependencies = [ "bottle>=0.12.16", "waitress>=1.3", - "doreah>=1.9.0, <2", + "doreah>=1.9.1, <2", "nimrodel>=0.8.0", "setproctitle>=1.1.10", #"pyvips>=2.1.16",