From c55e12dd4369a8c69bddfab501df862765cec85e Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 25 Apr 2022 03:24:16 +0200 Subject: [PATCH 1/4] Re-enabled cache per default --- dev/releases/branch.yml | 4 ++-- maloja/database/jinjaview.py | 3 ++- maloja/globalconf.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dev/releases/branch.yml b/dev/releases/branch.yml index 61415ea..f408e30 100644 --- a/dev/releases/branch.yml +++ b/dev/releases/branch.yml @@ -1,3 +1,3 @@ - "[Bugix] Improved signal handling" -- "[Bugix] Fixed constant re-caching of all-time stats" -- "[Performance] Disabled caches per default" +- "[Bugix] Fixed constant re-caching of all-time stats. significantly increasing page load speed" +- "[Logging] Disabled cache information when cache is not used" diff --git a/maloja/database/jinjaview.py b/maloja/database/jinjaview.py index ab19e73..80d1b67 100644 --- a/maloja/database/jinjaview.py +++ b/maloja/database/jinjaview.py @@ -23,7 +23,8 @@ class JinjaDBConnection: return self def __exit__(self, exc_type, exc_value, exc_traceback): self.conn.close() - log(f"Generated page with {self.hits}/{self.hits+self.misses} local Cache hits",module="debug_performance") + if malojaconfig['USE_REQUEST_CACHE']: + log(f"Generated page with {self.hits}/{self.hits+self.misses} local Cache hits",module="debug_performance") del self.cache def __getattr__(self,name): originalmethod = getattr(database,name) diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 2ab8f09..f150097 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -150,7 +150,7 @@ malojaconfig = Configuration( "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."), "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", False), - "use_global_cache":(tp.Boolean(), "Use global DB Cache", False) + "use_global_cache":(tp.Boolean(), "Use global DB Cache", True) }, "Fluff":{ "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"), From 62abc319303a6cb6463f7c27b6ef09b76fc67f86 Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 25 Apr 2022 03:37:12 +0200 Subject: [PATCH 2/4] Version bump --- dev/releases/3.0.yml | 6 ++++++ dev/releases/branch.yml | 4 +--- maloja/__pkginfo__.py | 2 +- pyproject.toml | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dev/releases/3.0.yml b/dev/releases/3.0.yml index 6a52c9d..9404100 100644 --- a/dev/releases/3.0.yml +++ b/dev/releases/3.0.yml @@ -42,3 +42,9 @@ minor_release_name: "Yeonhee" - "[Bugfix] Fixed importing a Spotify file without path" - "[Bugfix] No longer releasing database lock during scrobble creation" - "[Distribution] Experimental arm64 image" +3.0.7: + notes: + - "[Bugix] Improved signal handling" + - "[Bugix] Fixed constant re-caching of all-time stats, significantly increasing page load speed" + - "[Logging] Disabled cache information when cache is not used" + - "[Distribution] Experimental arm/v7 image" diff --git a/dev/releases/branch.yml b/dev/releases/branch.yml index f408e30..8b13789 100644 --- a/dev/releases/branch.yml +++ b/dev/releases/branch.yml @@ -1,3 +1 @@ -- "[Bugix] Improved signal handling" -- "[Bugix] Fixed constant re-caching of all-time stats. significantly increasing page load speed" -- "[Logging] Disabled cache information when cache is not used" + diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 44acbe8..de1dcb1 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "3.0.6" +VERSION = "3.0.7" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/pyproject.toml b/pyproject.toml index aa20899..fa9c34e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "3.0.6" +version = "3.0.7" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.6" From ad50ee866c33aa8e100b509835240d2cdfffcd22 Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 25 Apr 2022 04:28:53 +0200 Subject: [PATCH 3/4] More cache organization --- dev/releases/3.0.yml | 1 + dev/releases/branch.yml | 3 +- dev/write_tags.py | 1 + maloja/database/dbcache.py | 74 ++++++++++++++++++++++---------------- maloja/database/sqldb.py | 2 ++ maloja/globalconf.py | 2 +- 6 files changed, 51 insertions(+), 32 deletions(-) diff --git a/dev/releases/3.0.yml b/dev/releases/3.0.yml index 9404100..099b2c9 100644 --- a/dev/releases/3.0.yml +++ b/dev/releases/3.0.yml @@ -43,6 +43,7 @@ minor_release_name: "Yeonhee" - "[Bugfix] No longer releasing database lock during scrobble creation" - "[Distribution] Experimental arm64 image" 3.0.7: + commit: "62abc319303a6cb6463f7c27b6ef09b76fc67f86" notes: - "[Bugix] Improved signal handling" - "[Bugix] Fixed constant re-caching of all-time stats, significantly increasing page load speed" diff --git a/dev/releases/branch.yml b/dev/releases/branch.yml index 8b13789..00203d9 100644 --- a/dev/releases/branch.yml +++ b/dev/releases/branch.yml @@ -1 +1,2 @@ - +- "[Performance] Adjusted cache sizes" +- "[Logging] Added cache memory use information" diff --git a/dev/write_tags.py b/dev/write_tags.py index 4a3cb42..7b467a7 100644 --- a/dev/write_tags.py +++ b/dev/write_tags.py @@ -6,6 +6,7 @@ FOLDER = "dev/releases" releases = {} for f in os.listdir(FOLDER): + if f == "branch.yml": continue #maj,min = (int(i) for i in f.split('.')[:2]) with open(os.path.join(FOLDER,f)) as fd: diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index 57117d3..6bc81df 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -5,6 +5,7 @@ import lru import psutil import json +import sys from doreah.regular import runhourly from doreah.logging import log @@ -12,16 +13,10 @@ from ..globalconf import malojaconfig - - if malojaconfig['USE_GLOBAL_CACHE']: - CACHE_SIZE = 1000 - ENTITY_CACHE_SIZE = 100000 - cache = lru.LRU(CACHE_SIZE) - entitycache = lru.LRU(ENTITY_CACHE_SIZE) - - hits, misses = 0, 0 + cache = lru.LRU(10000) + entitycache = lru.LRU(100000) @@ -31,11 +26,10 @@ if malojaconfig['USE_GLOBAL_CACHE']: trim_cache() def print_stats(): - log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}") - #print("Full rundown:") - #import sys - #for k in cache.keys(): - # print(f"\t{k}\t{sys.getsizeof(cache[k])}") + for name,c in (('Cache',cache),('Entity Cache',entitycache)): + hits, misses = c.get_stats() + log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}") + log(f"System RAM Utilization: {psutil.virtual_memory().percent}%") def cached_wrapper(inner_func): @@ -49,12 +43,9 @@ if malojaconfig['USE_GLOBAL_CACHE']: global hits, misses key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) - if key in cache: - hits += 1 - return cache.get(key) - - else: - misses += 1 + try: + return cache[key] + except KeyError: result = inner_func(*args,**kwargs,dbconn=conn) cache[key] = result return result @@ -67,25 +58,18 @@ if malojaconfig['USE_GLOBAL_CACHE']: # cache that's aware of what we're calling def cached_wrapper_individual(inner_func): - def outer_func(set_arg,**kwargs): - - if 'dbconn' in kwargs: conn = kwargs.pop('dbconn') else: conn = None - #global hits, misses result = {} for id in set_arg: - if (inner_func,id) in entitycache: + try: result[id] = entitycache[(inner_func,id)] - #hits += 1 - else: + except KeyError: pass - #misses += 1 - remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn) for id in remaining: @@ -115,13 +99,14 @@ if malojaconfig['USE_GLOBAL_CACHE']: def trim_cache(): ramprct = psutil.virtual_memory().percent if ramprct > malojaconfig["DB_MAX_MEMORY"]: - log(f"{ramprct}% RAM usage, clearing cache and adjusting size!") + log(f"{ramprct}% RAM usage, clearing cache!") + for c in (cache,entitycache): + c.clear() #ratio = 0.6 #targetsize = max(int(len(cache) * ratio),50) #log(f"Reducing to {targetsize} entries") #cache.set_size(targetsize) #cache.set_size(HIGH_NUMBER) - cache.clear() #if cache.get_size() > CACHE_ADJUST_STEP: # cache.set_size(cache.get_size() - CACHE_ADJUST_STEP) @@ -156,3 +141,32 @@ def serialize(obj): elif isinstance(obj,dict): return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" return json.dumps(obj.hashable()) + + + +def get_size_of(obj,counted=None): + if counted is None: + counted = set() + if id(obj) in counted: return 0 + size = sys.getsizeof(obj) + counted.add(id(obj)) + try: + for k,v in obj.items(): + size += get_size_of(v,counted=counted) + except: + try: + for i in obj: + size += get_size_of(i,counted=counted) + except: + pass + return size + +def human_readable_size(obj): + units = ['','K','M','G','T','P'] + idx = 0 + bytes = get_size_of(obj) + while bytes > 1024 and len(units) > idx+1: + bytes = bytes / 1024 + idx += 1 + + return f"{bytes:.2f} {units[idx]}B" diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index 4ee49ca..496ab66 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -116,6 +116,8 @@ def connection_provider(func): with engine.connect() as connection: kwargs['dbconn'] = connection return func(*args,**kwargs) + + wrapper.__innerfunc__ = func return wrapper ##### DB <-> Dict translations diff --git a/maloja/globalconf.py b/maloja/globalconf.py index f150097..73515d6 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -148,7 +148,7 @@ malojaconfig = Configuration( "Technical":{ "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"), "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), - "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."), + "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 50, "RAM Usage in percent at which Maloja should no longer increase its database cache."), "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", False), "use_global_cache":(tp.Boolean(), "Use global DB Cache", True) }, From 7062c0b4404b6101647e9e850d7009a266c03e35 Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 25 Apr 2022 15:51:01 +0200 Subject: [PATCH 4/4] Update API.md --- API.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/API.md b/API.md index 5692116..3290a03 100644 --- a/API.md +++ b/API.md @@ -1,6 +1,7 @@ # Scrobbling -In order to scrobble from a wide selection of clients, you can use Maloja's standard-compliant APIs with the following settings: +Scrobbling can be done with the native API, see [below](#submitting-a-scrobble). +In order to scrobble from a wide selection of clients, you can also use Maloja's standard-compliant APIs with the following settings: GNU FM |   ------ | --------- @@ -41,7 +42,7 @@ The user starts playing '(Fine Layers of) Slaysenflite', which is exactly 3:00 m * If the user ends the play after 1:22, no scrobble is submitted * If the user ends the play after 2:06, a scrobble with `"duration":126` is submitted * If the user jumps back several times and ends the play after 3:57, a scrobble with `"duration":237` is submitted -* If the user jumps back several times and ends the play after 4:49, two scrobbles with `"duration":180` and `"duration":109` should be submitted +* If the user jumps back several times and ends the play after 4:49, two scrobbles with `"duration":180` and `"duration":109` are submitted @@ -54,11 +55,26 @@ The native Maloja API is reachable at `/apis/mlj_1`. Endpoints are listed on `/a All endpoints return JSON data. POST request can be made with query string or form data arguments, but this is discouraged - JSON should be used whenever possible. No application should ever rely on the non-existence of fields in the JSON data - i.e., additional fields can be added at any time without this being considered a breaking change. Existing fields should usually not be removed or changed, but it is always a good idea to add basic handling for missing fields. + +## Submitting a Scrobble + +The POST endpoint `/newscrobble` is used to submit new scrobbles. These use a flat JSON structure with the following fields: + +| Key | Type | Description | +| --- | --- | --- | +| `artists` | List(String) | Track artists | +| `title` | String | Track title | +| `album` | String | Name of the album (Optional) | +| `albumartists` | List(String) | Album artists (Optional) | +| `duration` | Integer | How long the song was listened to in seconds (Optional) | +| `length` | Integer | Actual length of the full song in seconds (Optional) | +| `time` | Integer | Timestamp of the listen if it was not at the time of submitting (Optional) | +| `nofix` | Boolean | Skip server-side metadata fixing (Optional) | ## General Structure - -Most endpoints follow this structure: +The API is not fully consistent in order to ensure backwards-compatibility. Refer to the individual endpoints. +Generally, most endpoints follow this structure: | Key | Type | Description | | --- | --- | --- | @@ -66,7 +82,7 @@ Most endpoints follow this structure: | `error` | Mapping | Details about the error if one occured. | | `warnings` | List | Any warnings that did not result in failure, but should be noted. Field is omitted if there are no warnings! | | `desc` | String | Human-readable feedback. This can be shown directly to the user if desired. | -| `list` | List | List of returned [entities](#Entity-Structure) | +| `list` | List | List of returned [entities](#entity-structure) | Both errors and warnings have the following structure: @@ -87,7 +103,7 @@ Whenever a list of entities is returned, they have the following fields: | Key | Type | Description | | --- | --- | --- | | `time` | Integer | Timestamp of the Scrobble in UTC | -| `track` | Mapping | The [track](#Track) being scrobbled | +| `track` | Mapping | The [track](#track) being scrobbled | | `duration` | Integer | How long the track was played for in seconds | | `origin` | String | Client that submitted the scrobble, or import source | @@ -118,7 +134,7 @@ Whenever a list of entities is returned, they have the following fields: | Key | Type | Description | | --- | --- | --- | -| `artists` | List | The [artists](#Artist) credited with the track | +| `artists` | List | The [artists](#artist) credited with the track | | `title` | String | The title of the track | | `length` | Integer | The full length of the track in seconds |