diff --git a/API.md b/API.md index 5692116..3290a03 100644 --- a/API.md +++ b/API.md @@ -1,6 +1,7 @@ # Scrobbling -In order to scrobble from a wide selection of clients, you can use Maloja's standard-compliant APIs with the following settings: +Scrobbling can be done with the native API, see [below](#submitting-a-scrobble). +In order to scrobble from a wide selection of clients, you can also use Maloja's standard-compliant APIs with the following settings: GNU FM |   ------ | --------- @@ -41,7 +42,7 @@ The user starts playing '(Fine Layers of) Slaysenflite', which is exactly 3:00 m * If the user ends the play after 1:22, no scrobble is submitted * If the user ends the play after 2:06, a scrobble with `"duration":126` is submitted * If the user jumps back several times and ends the play after 3:57, a scrobble with `"duration":237` is submitted -* If the user jumps back several times and ends the play after 4:49, two scrobbles with `"duration":180` and `"duration":109` should be submitted +* If the user jumps back several times and ends the play after 4:49, two scrobbles with `"duration":180` and `"duration":109` are submitted @@ -54,11 +55,26 @@ The native Maloja API is reachable at `/apis/mlj_1`. Endpoints are listed on `/a All endpoints return JSON data. POST request can be made with query string or form data arguments, but this is discouraged - JSON should be used whenever possible. No application should ever rely on the non-existence of fields in the JSON data - i.e., additional fields can be added at any time without this being considered a breaking change. Existing fields should usually not be removed or changed, but it is always a good idea to add basic handling for missing fields. + +## Submitting a Scrobble + +The POST endpoint `/newscrobble` is used to submit new scrobbles. These use a flat JSON structure with the following fields: + +| Key | Type | Description | +| --- | --- | --- | +| `artists` | List(String) | Track artists | +| `title` | String | Track title | +| `album` | String | Name of the album (Optional) | +| `albumartists` | List(String) | Album artists (Optional) | +| `duration` | Integer | How long the song was listened to in seconds (Optional) | +| `length` | Integer | Actual length of the full song in seconds (Optional) | +| `time` | Integer | Timestamp of the listen if it was not at the time of submitting (Optional) | +| `nofix` | Boolean | Skip server-side metadata fixing (Optional) | ## General Structure - -Most endpoints follow this structure: +The API is not fully consistent in order to ensure backwards-compatibility. Refer to the individual endpoints. +Generally, most endpoints follow this structure: | Key | Type | Description | | --- | --- | --- | @@ -66,7 +82,7 @@ Most endpoints follow this structure: | `error` | Mapping | Details about the error if one occured. | | `warnings` | List | Any warnings that did not result in failure, but should be noted. Field is omitted if there are no warnings! | | `desc` | String | Human-readable feedback. This can be shown directly to the user if desired. | -| `list` | List | List of returned [entities](#Entity-Structure) | +| `list` | List | List of returned [entities](#entity-structure) | Both errors and warnings have the following structure: @@ -87,7 +103,7 @@ Whenever a list of entities is returned, they have the following fields: | Key | Type | Description | | --- | --- | --- | | `time` | Integer | Timestamp of the Scrobble in UTC | -| `track` | Mapping | The [track](#Track) being scrobbled | +| `track` | Mapping | The [track](#track) being scrobbled | | `duration` | Integer | How long the track was played for in seconds | | `origin` | String | Client that submitted the scrobble, or import source | @@ -118,7 +134,7 @@ Whenever a list of entities is returned, they have the following fields: | Key | Type | Description | | --- | --- | --- | -| `artists` | List | The [artists](#Artist) credited with the track | +| `artists` | List | The [artists](#artist) credited with the track | | `title` | String | The title of the track | | `length` | Integer | The full length of the track in seconds | diff --git a/dev/releases/3.0.yml b/dev/releases/3.0.yml index 6a52c9d..099b2c9 100644 --- a/dev/releases/3.0.yml +++ b/dev/releases/3.0.yml @@ -42,3 +42,10 @@ minor_release_name: "Yeonhee" - "[Bugfix] Fixed importing a Spotify file without path" - "[Bugfix] No longer releasing database lock during scrobble creation" - "[Distribution] Experimental arm64 image" +3.0.7: + commit: "62abc319303a6cb6463f7c27b6ef09b76fc67f86" + notes: + - "[Bugix] Improved signal handling" + - "[Bugix] Fixed constant re-caching of all-time stats, significantly increasing page load speed" + - "[Logging] Disabled cache information when cache is not used" + - "[Distribution] Experimental arm/v7 image" diff --git a/dev/releases/3.1.yml b/dev/releases/3.1.yml index e24319d..b0ca543 100644 --- a/dev/releases/3.1.yml +++ b/dev/releases/3.1.yml @@ -6,3 +6,5 @@ minor_release_name: "Soyeon" - "[Feature] Implemented track title and artist name editing from web interface" - "[Feature] Implemented track and artist merging from web interface" - "[Feature] Implemented scrobble reparsing from web interface" + - "[Performance] Adjusted cache sizes" + - "[Logging] Added cache memory use information" diff --git a/dev/write_tags.py b/dev/write_tags.py index 4a3cb42..7b467a7 100644 --- a/dev/write_tags.py +++ b/dev/write_tags.py @@ -6,6 +6,7 @@ FOLDER = "dev/releases" releases = {} for f in os.listdir(FOLDER): + if f == "branch.yml": continue #maj,min = (int(i) for i in f.split('.')[:2]) with open(os.path.join(FOLDER,f)) as fd: diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 44acbe8..de1dcb1 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -4,7 +4,7 @@ # you know what f*ck it # this is hardcoded for now because of that damn project / package name discrepancy # i'll fix it one day -VERSION = "3.0.6" +VERSION = "3.0.7" HOMEPAGE = "https://github.com/krateng/maloja" diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index 02399f8..cd208ad 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -5,6 +5,7 @@ import lru import psutil import json +import sys from doreah.regular import runhourly from doreah.logging import log @@ -12,16 +13,10 @@ from ..pkg_global.conf import malojaconfig - - if malojaconfig['USE_GLOBAL_CACHE']: - CACHE_SIZE = 1000 - ENTITY_CACHE_SIZE = 100000 - cache = lru.LRU(CACHE_SIZE) - entitycache = lru.LRU(ENTITY_CACHE_SIZE) - - hits, misses = 0, 0 + cache = lru.LRU(10000) + entitycache = lru.LRU(100000) @@ -31,11 +26,10 @@ if malojaconfig['USE_GLOBAL_CACHE']: trim_cache() def print_stats(): - log(f"Cache Size: {len(cache)} [{len(entitycache)} E], System RAM Utilization: {psutil.virtual_memory().percent}%, Cache Hits: {hits}/{hits+misses}") - #print("Full rundown:") - #import sys - #for k in cache.keys(): - # print(f"\t{k}\t{sys.getsizeof(cache[k])}") + for name,c in (('Cache',cache),('Entity Cache',entitycache)): + hits, misses = c.get_stats() + log(f"{name}: Size: {len(c)} | Hits: {hits}/{hits+misses} | Estimated Memory: {human_readable_size(c)}") + log(f"System RAM Utilization: {psutil.virtual_memory().percent}%") def cached_wrapper(inner_func): @@ -49,12 +43,9 @@ if malojaconfig['USE_GLOBAL_CACHE']: global hits, misses key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) - if key in cache: - hits += 1 - return cache.get(key) - - else: - misses += 1 + try: + return cache[key] + except KeyError: result = inner_func(*args,**kwargs,dbconn=conn) cache[key] = result return result @@ -67,25 +58,18 @@ if malojaconfig['USE_GLOBAL_CACHE']: # cache that's aware of what we're calling def cached_wrapper_individual(inner_func): - def outer_func(set_arg,**kwargs): - - if 'dbconn' in kwargs: conn = kwargs.pop('dbconn') else: conn = None - #global hits, misses result = {} for id in set_arg: - if (inner_func,id) in entitycache: + try: result[id] = entitycache[(inner_func,id)] - #hits += 1 - else: + except KeyError: pass - #misses += 1 - remaining = inner_func(set(e for e in set_arg if e not in result),dbconn=conn) for id in remaining: @@ -115,13 +99,14 @@ if malojaconfig['USE_GLOBAL_CACHE']: def trim_cache(): ramprct = psutil.virtual_memory().percent if ramprct > malojaconfig["DB_MAX_MEMORY"]: - log(f"{ramprct}% RAM usage, clearing cache and adjusting size!") + log(f"{ramprct}% RAM usage, clearing cache!") + for c in (cache,entitycache): + c.clear() #ratio = 0.6 #targetsize = max(int(len(cache) * ratio),50) #log(f"Reducing to {targetsize} entries") #cache.set_size(targetsize) #cache.set_size(HIGH_NUMBER) - cache.clear() #if cache.get_size() > CACHE_ADJUST_STEP: # cache.set_size(cache.get_size() - CACHE_ADJUST_STEP) @@ -156,3 +141,32 @@ def serialize(obj): elif isinstance(obj,dict): return "{" + ",".join(serialize(o) + ":" + serialize(obj[o]) for o in obj) + "}" return json.dumps(obj.hashable()) + + + +def get_size_of(obj,counted=None): + if counted is None: + counted = set() + if id(obj) in counted: return 0 + size = sys.getsizeof(obj) + counted.add(id(obj)) + try: + for k,v in obj.items(): + size += get_size_of(v,counted=counted) + except: + try: + for i in obj: + size += get_size_of(i,counted=counted) + except: + pass + return size + +def human_readable_size(obj): + units = ['','K','M','G','T','P'] + idx = 0 + bytes = get_size_of(obj) + while bytes > 1024 and len(units) > idx+1: + bytes = bytes / 1024 + idx += 1 + + return f"{bytes:.2f} {units[idx]}B" diff --git a/maloja/database/jinjaview.py b/maloja/database/jinjaview.py index 37518ca..c0c48ee 100644 --- a/maloja/database/jinjaview.py +++ b/maloja/database/jinjaview.py @@ -23,7 +23,8 @@ class JinjaDBConnection: return self def __exit__(self, exc_type, exc_value, exc_traceback): self.conn.close() - log(f"Generated page with {self.hits}/{self.hits+self.misses} local Cache hits",module="debug_performance") + if malojaconfig['USE_REQUEST_CACHE']: + log(f"Generated page with {self.hits}/{self.hits+self.misses} local Cache hits",module="debug_performance") del self.cache def __getattr__(self,name): originalmethod = getattr(database,name) diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index 3848571..dcda72f 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -117,6 +117,8 @@ def connection_provider(func): with engine.connect() as connection: kwargs['dbconn'] = connection return func(*args,**kwargs) + + wrapper.__innerfunc__ = func return wrapper ##### DB <-> Dict translations diff --git a/maloja/pkg_global/conf.py b/maloja/pkg_global/conf.py index 31dc5e1..bbe1e7c 100644 --- a/maloja/pkg_global/conf.py +++ b/maloja/pkg_global/conf.py @@ -148,9 +148,9 @@ malojaconfig = Configuration( "Technical":{ "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"), "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), - "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 80, "RAM Usage in percent at which Maloja should no longer increase its database cache."), + "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 50, "RAM Usage in percent at which Maloja should no longer increase its database cache."), "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", False), - "use_global_cache":(tp.Boolean(), "Use global DB Cache", False) + "use_global_cache":(tp.Boolean(), "Use global DB Cache", True) }, "Fluff":{ "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"), diff --git a/pyproject.toml b/pyproject.toml index 38e3a1d..54d2787 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "malojaserver" -version = "3.0.6" +version = "3.0.7" description = "Self-hosted music scrobble database" readme = "./README.md" requires-python = ">=3.7"