diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 1c30f30..8e51d9e 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,7 +version = 2,4,8 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -21,7 +21,8 @@ requires = [ "wand>=0.5.4", "lesscpy>=0.13", "jinja2>2.11", - "lru-dict>=1.1.6" + "lru-dict>=1.1.6", + "psutil>0.5.0" ] resources = [ "web/*/*/*", diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 6c03ff8..589933c 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -29,6 +29,7 @@ USE_DB_CACHE = yes CACHE_DATABASE_SHORT = true CACHE_DATABASE_PERM = true #more permanent cache for old timeranges DB_CACHE_ENTRIES = 10000 #experiment with this depending on your RAM +DB_MAX_MEMORY = 75 # percentage of RAM utilization (whole container, not just maloja) that should trigger a flush INVALID_ARTISTS = ["[Unknown Artist]","Unknown Artist","Spotify"] REMOVE_FROM_TITLE = ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"] USE_PARSE_PLUGINS = no diff --git a/maloja/database.py b/maloja/database.py index dffc987..8e01297 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -34,6 +34,7 @@ from collections import namedtuple from threading import Lock import yaml import lru +import psutil # url handling from importlib.machinery import SourceFileLoader @@ -1055,6 +1056,7 @@ else: csz = settings.get_settings("DB_CACHE_ENTRIES") +cmp = settings.get_settings("DB_MAX_MEMORY") cache_query = lru.LRU(csz) cache_query_perm = lru.LRU(csz) @@ -1087,7 +1089,7 @@ def log_stats(): logstr = "{name}: {hitsperm} Perm Hits, {hitstmp} Tmp Hits, {misses} Misses; Current Size: {sizeperm}/{sizetmp}" for s in (cachestats["cache_query"],cachestats["cache_aggregate"]): log(logstr.format(name=s["name"],hitsperm=s["hits_perm"],hitstmp=s["hits_tmp"],misses=s["misses"], - sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"]))) + sizeperm=len(s["objperm"]),sizetmp=len(s["objtmp"])),module="debug") def db_query_cached(**kwargs): global cache_query, cache_query_perm @@ -1118,6 +1120,12 @@ def db_query_cached(**kwargs): result = db_query_full(**kwargs) if eligible_permanent_caching: cache_query_perm[key] = result elif eligible_temporary_caching: cache_query[key] = result + + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") + invalidate_caches() + return result @@ -1151,6 +1159,11 @@ def db_aggregate_cached(**kwargs): if eligible_permanent_caching: cache_aggregate_perm[key] = result elif eligible_temporary_caching: cache_aggregate[key] = result + ramprct = psutil.virtual_memory().percent + if ramprct > cmp: + log("{prct} RAM usage, dumping temporary caches!".format(prct=ramprct),module="debug") + invalidate_caches() + return result def invalidate_caches():