diff --git a/Dockerfile b/Dockerfile index 46f6e41..44c6960 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,18 @@ -FROM python:3.6-alpine +FROM python:3-alpine WORKDIR /usr/src/app -RUN apk update -RUN apk add gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers -RUN pip3 install psutil - -RUN pip3 install malojaserver +RUN apk add --no-cache --virtual .build-deps \ + gcc \ + libxml2-dev \ + libxslt-dev \ + py3-pip \ + libc-dev \ + linux-headers \ + && \ + pip3 install psutil && \ + pip3 install malojaserver && \ + apk del .build-deps EXPOSE 42010 diff --git a/README.md b/README.md index dcba5bb..185e2df 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ I can support you with issues best if you use **Alpine Linux**. In my experience 5) (Recommended) Until I have a proper service implemented, I would recommend setting two cronjobs for maloja: ``` -@reboot maloja start -42 0 * * * maloja restart +@reboot sleep 15 && maloja start +42 0 * * 2 maloja restart ``` diff --git a/install_alpine.sh b/install_alpine.sh index 75c9ac3..43705d7 100644 --- a/install_alpine.sh +++ b/install_alpine.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash -apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev +apk add python3 python3-dev gcc libxml2-dev libxslt-dev py3-pip libc-dev linux-headers +pip3 install psutil pip3 install malojaserver diff --git a/install_ubuntu.sh b/install_ubuntu.sh index ff0bd66..d30bd7b 100644 --- a/install_ubuntu.sh +++ b/install_ubuntu.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash apt update apt install python3 python3-pip +pip3 install psutil pip3 install malojaserver diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 0d9379c..f1bf847 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,4,9 +version = 2,4,13 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", @@ -15,7 +15,7 @@ links = { requires = [ "bottle>=0.12.16", "waitress>=1.3", - "doreah>=1.5.6", + "doreah>=1.6.3", "nimrodel>=0.6.3", "setproctitle>=1.1.10", "wand>=0.5.4", diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 427fe05..e6415d7 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -68,3 +68,4 @@ USE_PYHP = no #not recommended at the moment USE_JINJA = no #overwrites pyhp preference FEDERATION = yes #does nothing yet SKIP_SETUP = no +LOGGING = true diff --git a/maloja/database.py b/maloja/database.py index 9730588..ac94d77 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -933,6 +933,7 @@ def build_db(): log("Building database...") global SCROBBLES, ARTISTS, TRACKS + global TRACKS_NORMALIZED_SET, TRACKS_NORMALIZED, ARTISTS_NORMALIZED_SET, ARTISTS_NORMALIZED global SCROBBLESDICT, STAMPS SCROBBLES = [] @@ -941,6 +942,11 @@ def build_db(): STAMPS = [] SCROBBLESDICT = {} + TRACKS_NORMALIZED = [] + ARTISTS_NORMALIZED = [] + ARTISTS_NORMALIZED_SET = set() + TRACKS_NORMALIZED_SET = set() + # parse files db = tsv.parse_all(datadir("scrobbles"),"int","string","string",comments=False) @@ -1067,6 +1073,9 @@ cache_query_perm = lru.LRU(csz) cache_aggregate = lru.LRU(csz) cache_aggregate_perm = lru.LRU(csz) +perm_caching = settings.get_settings("CACHE_DATABASE_PERM") +temp_caching = settings.get_settings("CACHE_DATABASE_SHORT") + cachestats = { "cache_query":{ "hits_perm":0, @@ -1102,11 +1111,11 @@ def db_query_cached(**kwargs): eligible_permanent_caching = ( "timerange" in kwargs and not kwargs["timerange"].active() and - settings.get_settings("CACHE_DATABASE_PERM") + perm_caching ) eligible_temporary_caching = ( not eligible_permanent_caching and - settings.get_settings("CACHE_DATABASE_SHORT") + temp_caching ) # hit permanent cache for past timeranges @@ -1138,11 +1147,11 @@ def db_aggregate_cached(**kwargs): eligible_permanent_caching = ( "timerange" in kwargs and not kwargs["timerange"].active() and - settings.get_settings("CACHE_DATABASE_PERM") + perm_caching ) eligible_temporary_caching = ( not eligible_permanent_caching and - settings.get_settings("CACHE_DATABASE_SHORT") + temp_caching ) # hit permanent cache for past timeranges @@ -1173,17 +1182,18 @@ def invalidate_caches(): log("Database caches invalidated.") def reduce_caches(to=0.75): - global cache_query, cache_aggregate - for c in cache_query, cache_aggregate: + global cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm + for c in cache_query, cache_aggregate, cache_query_perm, cache_aggregate_perm: currentsize = len(c) - targetsize = int(currentsize * to) - c.set_size(targetsize) - c.set_size(csz) + if currentsize > 100: + targetsize = max(int(currentsize * to),10) + c.set_size(targetsize) + c.set_size(csz) def reduce_caches_if_low_ram(): ramprct = psutil.virtual_memory().percent if ramprct > cmp: - log("{prct}% RAM usage, reducing temporary caches!".format(prct=ramprct),module="debug") + log("{prct}% RAM usage, reducing caches!".format(prct=ramprct),module="debug") ratio = (cmp / ramprct) ** 3 reduce_caches(to=ratio) diff --git a/maloja/globalconf.py b/maloja/globalconf.py index 4118189..5ed516f 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -1,23 +1,34 @@ import os +from doreah.settings import get_settings +from doreah.settings import config as settingsconfig -# data folder -# must be determined first because getting settings relies on it -try: - DATA_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] - assert os.path.exists(DATA_DIR) -except: - DATA_DIR = os.path.join(os.environ["HOME"],".local/share/") +# check environment variables for data directory +# otherwise, go with defaults +setting_datadir = get_settings("DATA_DIRECTORY",files=[],environ_prefix="MALOJA_") +if setting_datadir is not None and os.path.exists(setting_datadir): + DATA_DIR = setting_datadir +else: + try: + HOME_DIR = os.environ["XDG_DATA_HOME"].split(":")[0] + assert os.path.exists(HOME_DIR) + except: + HOME_DIR = os.path.join(os.environ["HOME"],".local/share/") + + DATA_DIR = os.path.join(HOME_DIR,"maloja") -DATA_DIR = os.path.join(DATA_DIR,"maloja") os.makedirs(DATA_DIR,exist_ok=True) + + def datadir(*args): return os.path.join(DATA_DIR,*args) + + ### DOREAH CONFIGURATION from doreah import config @@ -26,9 +37,6 @@ config( pyhp={ "version": 2 }, - logging={ - "logfolder": datadir("logs") - }, settings={ "files":[ datadir("settings/default.ini"), @@ -44,9 +52,18 @@ config( } ) +# because we loaded a doreah module already before setting the config, we need to to that manually +settingsconfig._readpreconfig() + +config( + logging={ + "logfolder": datadir("logs") if get_settings("LOGGING") else None + } +) + +settingsconfig._readpreconfig() -from doreah.settings import get_settings # thumbor